diff --git a/source/isaaclab/changelog.d/mataylor-unmirror-cached-asset-paths.minor.rst b/source/isaaclab/changelog.d/mataylor-unmirror-cached-asset-paths.minor.rst new file mode 100644 index 00000000000..bc6067e80f5 --- /dev/null +++ b/source/isaaclab/changelog.d/mataylor-unmirror-cached-asset-paths.minor.rst @@ -0,0 +1,7 @@ +Added +^^^^^ + +* Added :func:`~isaaclab.utils.assets.unmirror_file_path`, which maps a locally cached asset copy + written by :func:`~isaaclab.utils.assets.retrieve_file_path` back to the URL it was downloaded + from. Exports of a stage that references cached copies can use it to name the source assets + instead of machine-specific cache paths. diff --git a/source/isaaclab/isaaclab/utils/assets.py b/source/isaaclab/isaaclab/utils/assets.py index 11ea04705bc..3a751edef7b 100644 --- a/source/isaaclab/isaaclab/utils/assets.py +++ b/source/isaaclab/isaaclab/utils/assets.py @@ -116,6 +116,10 @@ def _resolve_asset_root() -> str: _ANNOUNCED_MIRRORS: set[str] = set() """URLs already announced, so an asset consulted repeatedly is logged once.""" +_MIRRORED_URLS: dict[str, str] = {} +"""Source URL per locally cached copy, recorded as the copy is located rather than recovered +from its path, so a cache path is never inferred from a directory that merely looks like one.""" + _GIT_SSH_RE = re.compile(r"^[^@/:]+@[^:]+:.+") @@ -218,7 +222,10 @@ def _is_git_remote_path(git_path: str) -> bool: Returns: True if :paramref:`git_path` is a URL or SSH git path. """ - return bool(urlparse(git_path).scheme) or _GIT_SSH_RE.match(git_path) is not None + # ``urlparse`` reports a Windows drive letter as a scheme, so a local checkout such as + # ``C:\assets`` would otherwise be taken for a repository to clone. No URL scheme is a + # single character. + return len(urlparse(git_path).scheme) > 1 or _GIT_SSH_RE.match(git_path) is not None def _get_git_asset_repo_name(git_path: str) -> str: @@ -302,7 +309,32 @@ def _mirror_path(url: str, download_dir: str) -> str: return "" # ':' (port separator) is not a valid path character on Windows netloc = parsed.netloc.replace(":", "_") - return os.path.join(download_dir, parsed.scheme, netloc, *parsed.path.lstrip("/").split("/")) + mirrored = os.path.join(download_dir, parsed.scheme, netloc, *parsed.path.lstrip("/").split("/")) + # a host is what distinguishes a remote URL from a Windows drive letter, which ``urlparse`` + # also reports as a scheme + if parsed.netloc: + _MIRRORED_URLS[os.path.abspath(mirrored)] = url + return mirrored + + +def unmirror_file_path(path: str) -> str: + """Maps a locally cached asset copy back to the URL it was downloaded from. + + :func:`retrieve_file_path` hands callers a local copy of a remote asset, so a stage built + from one records a path that only resolves on the machine holding the cache. An export of + that stage can use this to name the source asset instead. + + Only copies this process located are known, so a locally authored path is never mistaken + for a cached copy. A copy mirrored by an earlier run is still recognised, because retrieval + walks the whole dependency tree even when every file is already cached. + + Args: + path: Local filesystem path, typically an asset path read from a USD layer. + + Returns: + The URL the copy was cached from, or ``""`` when this process did not cache it. + """ + return _MIRRORED_URLS.get(os.path.abspath(path), "") def _remote_fingerprint(url: str) -> dict | None: diff --git a/source/isaaclab/test/utils/test_assets.py b/source/isaaclab/test/utils/test_assets.py index 85fed70d19d..1aaac2764a9 100644 --- a/source/isaaclab/test/utils/test_assets.py +++ b/source/isaaclab/test/utils/test_assets.py @@ -9,6 +9,7 @@ import importlib import json import logging +import os from pathlib import Path from types import SimpleNamespace @@ -258,6 +259,22 @@ def fail_run_git_command(command): assert (asset_path / "example_bot.usd").read_text(encoding="utf-8") == "#usda 1.0\n" +@pytest.mark.parametrize( + ("git_path", "is_remote"), + [ + ("https://example.com/example-assets.git", True), + ("git@example.com:org/example-assets.git", True), + ("/home/user/newton-assets", False), + # ``urlparse`` reports a drive letter as a scheme, so these read as remote repositories + ("C:/Users/user/newton-assets", False), + (r"C:\Users\user\newton-assets", False), + ], +) +def test_git_asset_paths_tell_a_windows_drive_letter_from_a_url_scheme(git_path, is_remote): + """Test a local Windows checkout is not mistaken for a repository to clone into the cache.""" + assert assets_utils._is_git_remote_path(git_path) is is_remote + + def test_retrieve_git_asset_path_raises_for_missing_asset(tmp_path): """Test that git asset retrieval raises when the requested asset is missing.""" repo_dir = tmp_path / "newton-assets" @@ -277,6 +294,7 @@ def asset_cache(tmp_path, monkeypatch): monkeypatch.setattr(assets_utils, "_REMOTE_FINGERPRINTS", {}) monkeypatch.setattr(assets_utils, "_ANNOUNCED_MIRROR_DIRS", set()) monkeypatch.setattr(assets_utils, "_ANNOUNCED_MIRRORS", set()) + monkeypatch.setattr(assets_utils, "_MIRRORED_URLS", {}) return tmp_path @@ -419,6 +437,69 @@ def test_using_local_copies_is_announced_once_per_cache_directory(asset_cache, m assert {_REMOTE_URL, other_url} == {record.args[0] for record in per_asset} +@pytest.mark.parametrize( + "url", + [ + "https://example.com/Assets/Isaac/6.0/Isaac/Props/Blocks/DexCube/Materials/dex_cube_mod.png", + "http://example.com/Assets/example.usd", + "omniverse://nucleus.example-lab.com:3009/Assets/example.usd", + ], +) +def test_unmirror_file_path_recovers_the_url_a_copy_was_cached_from(asset_cache, url): + """Test a cached copy names the asset it came from, so exports do not carry local paths.""" + assert assets_utils.unmirror_file_path(assets_utils._mirror_path(url, str(asset_cache))) == url + + +@pytest.mark.parametrize( + "path", + ["/home/user/assets/example.usd", "Materials/dex_cube_mod.png", "OmniPBR.mdl", ""], +) +def test_unmirror_file_path_leaves_paths_outside_the_cache_unclaimed(asset_cache, path): + """Test a locally authored asset path is not mistaken for a cached remote copy.""" + assert assets_utils.unmirror_file_path(path) == "" + + +@pytest.mark.parametrize( + "path", + [ + # ``Omniverse`` is where Omniverse puts user projects by default + "C:/Users/user/Omniverse/MyProject/scene.usd", + "/data/omniverse/assets/robot.usd", + "/mnt/nfs/OMNIVERSE/Library/Wood/oak.mdl", + "/home/user/projects/https/site/logo.png", + ], +) +def test_unmirror_file_path_leaves_a_directory_named_after_a_url_scheme_unclaimed(asset_cache, path): + """Test an ordinary local layout is not read as a cache layout because of a directory name.""" + assert assets_utils.unmirror_file_path(path) == "" + + +def test_unmirror_file_path_does_not_claim_a_windows_drive_letter_path(asset_cache): + """Test a drive letter, which ``urlparse`` also reports as a scheme, is not read as a URL.""" + mirrored = assets_utils._mirror_path("C:/Users/user/assets/robot.usd", str(asset_cache)) + + assert assets_utils.unmirror_file_path(mirrored) == "" + + +def test_unmirror_file_path_recognises_a_copy_reported_with_forward_slashes(asset_cache): + """Test a copy is recognised when USD reports it with forward slashes, as it does on Windows.""" + url = "https://example.com/Assets/Isaac/example.usd" + mirrored = assets_utils._mirror_path(url, str(asset_cache)) + + assert assets_utils.unmirror_file_path(mirrored.replace(os.sep, "/")) == url + + +def test_unmirror_file_path_recognises_a_copy_cached_by_an_earlier_run(asset_cache, monkeypatch): + """Test a warm cache still names its source, since no download happens to record it.""" + revision = {"hash": "abc123", "version": "", "size": 12, "modified_time": "2026-07-01 10:00:00"} + mirrored = _cache_asset(asset_cache, _REMOTE_URL, b"cached bytes", revision) + # no payload is served, so the URL is recovered without the asset being downloaded again + _serve(monkeypatch, {_REMOTE_URL: revision}) + assets_utils.read_file(_REMOTE_URL) + + assert assets_utils.unmirror_file_path(str(mirrored)) == _REMOTE_URL + + def test_newton_asset_dir_uses_environment_override(tmp_path, monkeypatch): """Test that the Newton asset directory is defined from the environment.""" repo_dir = tmp_path / "newton-assets" diff --git a/source/isaaclab_tasks/changelog.d/mataylor-unmirror-cached-asset-paths.skip b/source/isaaclab_tasks/changelog.d/mataylor-unmirror-cached-asset-paths.skip new file mode 100644 index 00000000000..1c7429d83e4 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/mataylor-unmirror-cached-asset-paths.skip @@ -0,0 +1,3 @@ +Rendering test stage dumps rewrite cached asset paths back to their source URLs, so a stage saved +through ``ISAAC_LAB_SAVE_STAGES`` no longer carries texture paths that only resolve on the machine +that ran the test. diff --git a/source/isaaclab_tasks/test/rendering_test_utils.py b/source/isaaclab_tasks/test/rendering_test_utils.py index 654f0bdaea6..e74032892cd 100644 --- a/source/isaaclab_tasks/test/rendering_test_utils.py +++ b/source/isaaclab_tasks/test/rendering_test_utils.py @@ -22,6 +22,8 @@ from isaaclab.utils.warp import ProxyArray if TYPE_CHECKING: + from pxr import Sdf + from isaaclab.sensors.camera import CameraData logger = logging.getLogger(__name__) @@ -510,6 +512,20 @@ def _sanitize_golden_stage_text(text: str) -> str: return text.rstrip("\n") + "\n" +def _restore_remote_asset_paths(layer: "Sdf.Layer") -> None: + """Point cached asset paths in ``layer`` back at the URLs they were downloaded from. + + Remote USD assets are referenced through a local cache copy, so flattening resolves the + textures and materials they carry into absolute cache paths that exist only on the machine + that ran the test. Locally authored paths are left untouched. + """ + from pxr import UsdUtils # noqa: PLC0415 + + from isaaclab.utils.assets import unmirror_file_path # noqa: PLC0415 + + UsdUtils.ModifyAssetPaths(layer, lambda asset_path: unmirror_file_path(asset_path) or asset_path) + + def maybe_save_stage( test_name: str, physics_backend: str, @@ -551,6 +567,7 @@ def maybe_save_stage( flat_layer = opened_stage.Flatten() if flat_layer is None: pytest.fail(f"Could not flatten the saved stage at {stage_path}.") + _restore_remote_asset_paths(flat_layer) if out_dir: os.makedirs(out_dir, exist_ok=True)