Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Added
^^^^^

* Added :func:`~isaaclab.utils.assets.unmirror_file_path`, which maps a locally cached asset copy

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 Suggestion · Api — Changelog omits git remote-path behavior change

The fragment documents only the new unmirror_file_path, but the _is_git_remote_path change also alters user-visible behavior of the public retrieve_git_asset_path: a Windows local checkout such as C:\assets was previously treated as a remote repository and cloned into the cache, and is now used in place. Repository rules require changed behavior to appear in the release note; add a Fixed entry covering it.

written by :func:`~isaaclab.utils.assets.retrieve_file_path` back to the URL it was downloaded
from. Exports of a stage that references cached copies can use it to name the source assets
instead of machine-specific cache paths.
36 changes: 34 additions & 2 deletions source/isaaclab/isaaclab/utils/assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@ def _resolve_asset_root() -> str:
_ANNOUNCED_MIRRORS: set[str] = set()
"""URLs already announced, so an asset consulted repeatedly is logged once."""

_MIRRORED_URLS: dict[str, str] = {}
"""Source URL per locally cached copy, recorded as the copy is located rather than recovered
from its path, so a cache path is never inferred from a directory that merely looks like one."""

_GIT_SSH_RE = re.compile(r"^[^@/:]+@[^:]+:.+")


Expand Down Expand Up @@ -218,7 +222,10 @@ def _is_git_remote_path(git_path: str) -> bool:
Returns:
True if :paramref:`git_path` is a URL or SSH git path.
"""
return bool(urlparse(git_path).scheme) or _GIT_SSH_RE.match(git_path) is not None
# ``urlparse`` reports a Windows drive letter as a scheme, so a local checkout such as
# ``C:\assets`` would otherwise be taken for a repository to clone. No URL scheme is a
# single character.
return len(urlparse(git_path).scheme) > 1 or _GIT_SSH_RE.match(git_path) is not None


def _get_git_asset_repo_name(git_path: str) -> str:
Expand Down Expand Up @@ -302,7 +309,32 @@ def _mirror_path(url: str, download_dir: str) -> str:
return ""
# ':' (port separator) is not a valid path character on Windows
netloc = parsed.netloc.replace(":", "_")
return os.path.join(download_dir, parsed.scheme, netloc, *parsed.path.lstrip("/").split("/"))
mirrored = os.path.join(download_dir, parsed.scheme, netloc, *parsed.path.lstrip("/").split("/"))
# a host is what distinguishes a remote URL from a Windows drive letter, which ``urlparse``
# also reports as a scheme
if parsed.netloc:
_MIRRORED_URLS[os.path.abspath(mirrored)] = url
return mirrored


def unmirror_file_path(path: str) -> str:
"""Maps a locally cached asset copy back to the URL it was downloaded from.

:func:`retrieve_file_path` hands callers a local copy of a remote asset, so a stage built
from one records a path that only resolves on the machine holding the cache. An export of
that stage can use this to name the source asset instead.

Only copies this process located are known, so a locally authored path is never mistaken
for a cached copy. A copy mirrored by an earlier run is still recognised, because retrieval
walks the whole dependency tree even when every file is already cached.

Args:
path: Local filesystem path, typically an asset path read from a USD layer.

Returns:
The URL the copy was cached from, or ``""`` when this process did not cache it.
"""
return _MIRRORED_URLS.get(os.path.abspath(path), "")


def _remote_fingerprint(url: str) -> dict | None:
Expand Down
76 changes: 76 additions & 0 deletions source/isaaclab/test/utils/test_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import importlib
import json
import logging
import os
from pathlib import Path
from types import SimpleNamespace

Expand Down Expand Up @@ -258,6 +259,22 @@ def fail_run_git_command(command):
assert (asset_path / "example_bot.usd").read_text(encoding="utf-8") == "#usda 1.0\n"


@pytest.mark.parametrize(
("git_path", "is_remote"),
[
("https://example.com/example-assets.git", True),
("git@example.com:org/example-assets.git", True),
("/home/user/newton-assets", False),
# ``urlparse`` reports a drive letter as a scheme, so these read as remote repositories
("C:/Users/user/newton-assets", False),
(r"C:\Users\user\newton-assets", False),
],
)
def test_git_asset_paths_tell_a_windows_drive_letter_from_a_url_scheme(git_path, is_remote):
"""Test a local Windows checkout is not mistaken for a repository to clone into the cache."""
assert assets_utils._is_git_remote_path(git_path) is is_remote


def test_retrieve_git_asset_path_raises_for_missing_asset(tmp_path):
"""Test that git asset retrieval raises when the requested asset is missing."""
repo_dir = tmp_path / "newton-assets"
Expand All @@ -277,6 +294,7 @@ def asset_cache(tmp_path, monkeypatch):
monkeypatch.setattr(assets_utils, "_REMOTE_FINGERPRINTS", {})
monkeypatch.setattr(assets_utils, "_ANNOUNCED_MIRROR_DIRS", set())
monkeypatch.setattr(assets_utils, "_ANNOUNCED_MIRRORS", set())
monkeypatch.setattr(assets_utils, "_MIRRORED_URLS", {})
return tmp_path


Expand Down Expand Up @@ -419,6 +437,64 @@ def test_using_local_copies_is_announced_once_per_cache_directory(asset_cache, m
assert {_REMOTE_URL, other_url} == {record.args[0] for record in per_asset}


@pytest.mark.parametrize(
"url",
[
"https://example.com/Assets/Isaac/6.0/Isaac/Props/Blocks/DexCube/Materials/dex_cube_mod.png",
"http://example.com/Assets/example.usd",
"omniverse://nucleus.example-lab.com:3009/Assets/example.usd",
],
)
def test_unmirror_file_path_recovers_the_url_a_copy_was_cached_from(asset_cache, url):
"""Test a cached copy names the asset it came from, so exports do not carry local paths."""
assert assets_utils.unmirror_file_path(assets_utils._mirror_path(url, str(asset_cache))) == url


@pytest.mark.parametrize(
"path",
[
"/home/user/assets/example.usd",
"Materials/dex_cube_mod.png",
"OmniPBR.mdl",
"",
# a directory named after a URL scheme is an ordinary local layout -- ``Omniverse`` is
# where Omniverse puts user projects by default -- and must not read as a cached copy
"/data/omniverse/assets/robot.usd",
"C:/Users/user/Omniverse/MyProject/scene.usd",
"/home/user/projects/https/site/logo.png",
],
)
def test_unmirror_file_path_leaves_paths_the_cache_did_not_write_unclaimed(asset_cache, path):
"""Test a locally authored asset path is not mistaken for a cached remote copy."""
assert assets_utils.unmirror_file_path(path) == ""


def test_unmirror_file_path_does_not_claim_a_windows_drive_letter_path(asset_cache):
"""Test a drive letter, which ``urlparse`` also reports as a scheme, is not read as a URL."""
mirrored = assets_utils._mirror_path("C:/Users/user/assets/robot.usd", str(asset_cache))

assert assets_utils.unmirror_file_path(mirrored) == ""


def test_unmirror_file_path_recognises_a_copy_reported_with_forward_slashes(asset_cache):
"""Test a copy is recognised when USD reports it with forward slashes, as it does on Windows."""
url = "https://example.com/Assets/Isaac/example.usd"
mirrored = assets_utils._mirror_path(url, str(asset_cache))

assert assets_utils.unmirror_file_path(mirrored.replace(os.sep, "/")) == url


def test_unmirror_file_path_recognises_a_copy_cached_by_an_earlier_run(asset_cache, monkeypatch):
"""Test a warm cache still names its source, since no download happens to record it."""
revision = {"hash": "abc123", "version": "", "size": 12, "modified_time": "2026-07-01 10:00:00"}
mirrored = _cache_asset(asset_cache, _REMOTE_URL, b"cached bytes", revision)
# no payload is served, so the URL is recovered without the asset being downloaded again
_serve(monkeypatch, {_REMOTE_URL: revision})
assets_utils.read_file(_REMOTE_URL)

assert assets_utils.unmirror_file_path(str(mirrored)) == _REMOTE_URL


def test_newton_asset_dir_uses_environment_override(tmp_path, monkeypatch):
"""Test that the Newton asset directory is defined from the environment."""
repo_dir = tmp_path / "newton-assets"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Rendering test stage dumps rewrite cached asset paths back to their source URLs, so a stage saved
through ``ISAAC_LAB_SAVE_STAGES`` no longer carries texture paths that only resolve on the machine
that ran the test.
17 changes: 17 additions & 0 deletions source/isaaclab_tasks/test/rendering_test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
from isaaclab.utils.warp import ProxyArray

if TYPE_CHECKING:
from pxr import Sdf

from isaaclab.sensors.camera import CameraData

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -510,6 +512,20 @@ def _sanitize_golden_stage_text(text: str) -> str:
return text.rstrip("\n") + "\n"


def _restore_remote_asset_paths(layer: "Sdf.Layer") -> None:
"""Point cached asset paths in ``layer`` back at the URLs they were downloaded from.

Remote USD assets are referenced through a local cache copy, so flattening resolves the
textures and materials they carry into absolute cache paths that exist only on the machine
that ran the test. Locally authored paths are left untouched.
"""
from pxr import UsdUtils # noqa: PLC0415

from isaaclab.utils.assets import unmirror_file_path # noqa: PLC0415

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this be moved to the top?

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We already have imports at the top of isaaclab.utils, I would assume yes? i wonder why would we want to delay this import?


UsdUtils.ModifyAssetPaths(layer, lambda asset_path: unmirror_file_path(asset_path) or asset_path)


def maybe_save_stage(
test_name: str,
physics_backend: str,
Expand Down Expand Up @@ -551,6 +567,7 @@ def maybe_save_stage(
flat_layer = opened_stage.Flatten()
if flat_layer is None:
pytest.fail(f"Could not flatten the saved stage at {stage_path}.")
_restore_remote_asset_paths(flat_layer)

if out_dir:
os.makedirs(out_dir, exist_ok=True)
Expand Down
Loading