From bd47fbbb6c2160ac041cdf4c5d4479c6f1e8664c Mon Sep 17 00:00:00 2001 From: Shao Su Date: Mon, 3 Aug 2026 17:45:16 -0700 Subject: [PATCH] feat(oak): make oak frame tracker track 1 stream Signed-off-by: Shao Su --- docs/source/device/oak.rst | 21 +++--- docs/source/device/trackers.rst | 5 +- examples/oxr/python/test_oak_camera.py | 45 +++++++------ examples/schemaio/frame_metadata_printer.cpp | 59 ++++++----------- .../frame_metadata_tracker_oak_base.hpp | 4 +- .../cpp/frame_metadata_tracker_oak.cpp | 27 ++------ .../frame_metadata_tracker_oak.hpp | 66 ++++++------------- .../python/tracker_bindings.cpp | 21 +++--- .../cpp/live_deviceio_factory.cpp | 2 +- .../live_frame_metadata_tracker_oak_impl.cpp | 59 +++++------------ .../live_frame_metadata_tracker_oak_impl.hpp | 16 ++--- .../mcap/cpp/inc/mcap/recording_traits.hpp | 1 + src/plugins/oak/README.md | 21 +++--- 13 files changed, 133 insertions(+), 214 deletions(-) diff --git a/docs/source/device/oak.rst b/docs/source/device/oak.rst index ccc37c59a..4d16d98fe 100644 --- a/docs/source/device/oak.rst +++ b/docs/source/device/oak.rst @@ -108,10 +108,10 @@ controllers, etc.): :doc:`TeleopSession <../getting_started/teleop_session>`'s ``PluginConfig``, passing ``--collection-prefix`` so the plugin pushes metadata via OpenXR ``SchemaPusher``. -2. **Host tracker** — create ``FrameMetadataTrackerOak`` with the **same** - collection prefix and stream list (see :doc:`trackers`). TeleopSession's - DeviceIO layer uses the live tracker implementation to read the pushed - tensors and write MCAP channels. +2. **Host trackers** — create one ``FrameMetadataTrackerOak`` per stream, each + with a ``collection_id`` matching the plugin's ``{collection_prefix}/{StreamName}`` + (see :doc:`trackers`). TeleopSession's DeviceIO layer uses the live tracker + implementation to read the pushed tensors and write MCAP channels. 3. **MCAP config** — add the tracker to both ``TeleopSessionConfig.trackers`` and ``McapRecordingConfig.tracker_names``. See also :doc:`../references/mcap_record_replay`. @@ -120,22 +120,25 @@ controllers, etc.): from pathlib import Path - from isaacteleop.deviceio import FrameMetadataTrackerOak, McapRecordingConfig, StreamType + from isaacteleop.deviceio import FrameMetadataTrackerOak, McapRecordingConfig from isaacteleop.teleop_session_manager import PluginConfig, TeleopSession, TeleopSessionConfig PLUGIN_ROOT = Path("build/src/plugins") # or your installed plugin search path COLLECTION_PREFIX = "oak_camera" - STREAMS = [StreamType.Color, StreamType.MonoLeft] + STREAM_NAMES = ["Color", "MonoLeft"] - oak_tracker = FrameMetadataTrackerOak(COLLECTION_PREFIX, STREAMS) + oak_trackers = [ + FrameMetadataTrackerOak(f"{COLLECTION_PREFIX}/{name}") + for name in STREAM_NAMES + ] config = TeleopSessionConfig( app_name="OakTeleop", pipeline=pipeline, # your retargeting pipeline - trackers=[oak_tracker], + trackers=oak_trackers, mcap_config=McapRecordingConfig( "recording.mcap", - [(oak_tracker, "oak_metadata")], + [(t, f"oak_metadata/{name}") for t, name in zip(oak_trackers, STREAM_NAMES)], ), plugins=[ PluginConfig( diff --git a/docs/source/device/trackers.rst b/docs/source/device/trackers.rst index 3334ff0f0..643d69a54 100644 --- a/docs/source/device/trackers.rst +++ b/docs/source/device/trackers.rst @@ -220,14 +220,15 @@ reads the PICO ``XR_BD_body_tracking`` extension directly. FrameMetadataTrackerOak ~~~~~~~~~~~~~~~~~~~~~~~ -Multi-channel tracker for per-frame metadata from OAK camera streams. +Single-stream tracker for per-frame metadata from an OAK camera stream. +Create one instance per stream (e.g. ``"oak_camera/Color"``, ``"oak_camera/MonoLeft"``). Uses the :code-file:`SchemaTracker ` utility internally. - Schema: :code-file:`src/core/schema/fbs/oak.fbs` - C++ header: ``#include `` - Python import: ``from isaacteleop.deviceio import FrameMetadataTrackerOak`` -- Record channels: one per configured stream (e.g. ``Color``, ``MonoLeft``) | MCAP schema: ``core.FrameMetadataOakRecord`` +- Record channels: ``frame_metadata`` | MCAP schema: ``core.FrameMetadataOakRecord`` - Tests: - :code-file:`src/core/schema_tests/cpp/test_oak.cpp` diff --git a/examples/oxr/python/test_oak_camera.py b/examples/oxr/python/test_oak_camera.py index 6647e4092..7c70cb8ea 100755 --- a/examples/oxr/python/test_oak_camera.py +++ b/examples/oxr/python/test_oak_camera.py @@ -26,7 +26,9 @@ import isaacteleop.deviceio as deviceio import isaacteleop.oxr as oxr -PLUGIN_ROOT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "plugins" +PLUGIN_ROOT_DIR = ( + Path(__file__).resolve().parent.parent.parent.parent / "install/plugins" +) MODE_NO_METADATA = "no-metadata" MODE_SCHEMA_PUSHER = "schema-pusher" @@ -55,21 +57,18 @@ def _run_recording_loop(plugin, duration: float): def _run_schema_pusher( plugin, duration: float, - tracker, + trackers: list, stream_names: list[str], required_extensions: list[str], - mcap_filename: str, + recording_config: deviceio.McapRecordingConfig, ): """Read metadata via OpenXR schema tracker and record to MCAP on the host side.""" with oxr.OpenXRSession("OakCameraTest", required_extensions) as oxr_session: handles = oxr_session.get_handles() print(" ✓ OpenXR session created") - recording_config = deviceio.McapRecordingConfig( - mcap_filename, [(tracker, "oak_metadata")] - ) with deviceio.DeviceIOSession.run( - [tracker], handles, recording_config + trackers, handles, recording_config ) as session: print(" ✓ DeviceIO session initialized (recording active during update())") print() @@ -88,8 +87,8 @@ def _run_schema_pusher( frame_count += 1 elapsed = time.time() - start_time - for idx, name in enumerate(stream_names): - tracked = tracker.get_stream_data(session, idx) + for tracker, name in zip(trackers, stream_names): + tracked = tracker.get_data(session) if ( tracked.data is not None and tracked.data.sequence_number != last_seq.get(name, -1) @@ -152,23 +151,23 @@ def run_test(duration: float = 10.0, mode: str = MODE_NO_METADATA): # 3. Prepare mode-specific state stream_names = ["Color", "MonoLeft"] - stream_types = [deviceio.StreamType.Color, deviceio.StreamType.MonoLeft] collection_prefix = "oak_camera" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") mcap_filename = f"camera_metadata_{timestamp}.mcap" - tracker = None + trackers = [] required_extensions = [] if mode == MODE_SCHEMA_PUSHER: - print("[Step 3] Creating composite FrameMetadataTrackerOak...") - tracker = deviceio.FrameMetadataTrackerOak(collection_prefix, stream_types) + print("[Step 3] Creating FrameMetadataTrackerOak instances (one per stream)...") + trackers = [ + deviceio.FrameMetadataTrackerOak(f"{collection_prefix}/{name}") + for name in stream_names + ] print( - f" Created tracker (prefix: {collection_prefix}, streams: {stream_names})" - ) - required_extensions = deviceio.DeviceIOSession.get_required_extensions( - [tracker] + f" Created {len(trackers)} trackers: {[f'{collection_prefix}/{n}' for n in stream_names]}" ) + required_extensions = deviceio.DeviceIOSession.get_required_extensions(trackers) print() print("[Step 4] Getting required OpenXR extensions...") print(f" Required extensions: {required_extensions}") @@ -207,13 +206,21 @@ def run_test(duration: float = 10.0, mode: str = MODE_NO_METADATA): print(" Camera plugin started") if mode == MODE_SCHEMA_PUSHER: + recording_config = deviceio.McapRecordingConfig( + mcap_filename, + [ + (t, f"oak_metadata/{name}") + for t, name in zip(trackers, stream_names) + ], + ) + _run_schema_pusher( plugin, duration, - tracker, + trackers, stream_names, required_extensions, - mcap_filename, + recording_config, ) else: _run_recording_loop(plugin, duration) diff --git a/examples/schemaio/frame_metadata_printer.cpp b/examples/schemaio/frame_metadata_printer.cpp index b7ce5ac2e..c57e9f3d1 100644 --- a/examples/schemaio/frame_metadata_printer.cpp +++ b/examples/schemaio/frame_metadata_printer.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -67,39 +68,40 @@ try std::cout << "Frame Metadata Printer (prefix: " << collection_prefix << ")" << std::endl; - // Track all three stream types; streams without a pusher simply won't receive data. - std::vector streams = { core::StreamType_Color, core::StreamType_MonoLeft, - core::StreamType_MonoRight }; + // One tracker per stream; streams without a pusher simply won't receive data. + const std::vector collection_ids = { + collection_prefix + "/Color", + collection_prefix + "/MonoLeft", + collection_prefix + "/MonoRight", + }; - std::cout << "[Step 1] Creating FrameMetadataTrackerOak..." << std::endl; - auto tracker = std::make_shared(collection_prefix, streams, MAX_FLATBUFFER_SIZE); + std::cout << "[Step 1] Creating FrameMetadataTrackerOak instances..." << std::endl; + std::vector> trackers; + trackers.reserve(collection_ids.size()); + for (const auto& cid : collection_ids) + trackers.push_back(std::make_shared(cid, MAX_FLATBUFFER_SIZE)); std::cout << "[Step 2] Creating OpenXR session with required extensions..." << std::endl; - std::vector> trackers = { tracker }; - auto required_extensions = core::DeviceIOSession::get_required_extensions(trackers); + std::vector> itracker_list(trackers.begin(), trackers.end()); + auto required_extensions = core::DeviceIOSession::get_required_extensions(itracker_list); auto oxr_session = std::make_shared("FrameMetadataPrinter", required_extensions); std::cout << " OpenXR session created" << std::endl; std::cout << "[Step 3] Creating DeviceIOSession..." << std::endl; - auto session = core::DeviceIOSession::run(trackers, oxr_session->get_handles()); + auto session = core::DeviceIOSession::run(itracker_list, oxr_session->get_handles()); std::cout << "[Step 4] Reading samples (press Ctrl+C to stop)..." << std::endl; size_t received_count = 0; - // Per-stream last-seen sequence number. nullopt means the stream has never + // Per-tracker last-seen sequence number. nullopt means the stream has never // been observed — the first sample is always printed regardless of its value. - // If data is already present at startup we seed with its sequence so we don't - // reprint it; absent data stays nullopt so sequence 0 is never skipped. - size_t stream_count = tracker->get_stream_count(); - std::vector> last_sequences(stream_count); - for (size_t i = 0; i < stream_count; ++i) + std::vector> last_sequences(trackers.size()); + for (size_t i = 0; i < trackers.size(); ++i) { - const auto& tracked = tracker->get_stream_data(*session, i); + const auto& tracked = trackers[i]->get_data(*session); if (tracked.data) - { last_sequences[i] = tracked.data->sequence_number; - } } auto last_status_time = std::chrono::steady_clock::now(); @@ -109,28 +111,9 @@ try { session->update(); - // Refresh stream count and extend per-stream tracking if streams were added. - stream_count = tracker->get_stream_count(); - if (last_sequences.size() != stream_count) - { - size_t old_count = last_sequences.size(); - last_sequences.resize(stream_count); - // Newly added streams start as nullopt; seed with current sequence if - // data is already present so we don't reprint an existing sample. - for (size_t i = old_count; i < stream_count; ++i) - { - const auto& tracked = tracker->get_stream_data(*session, i); - if (tracked.data) - { - last_sequences[i] = tracked.data->sequence_number; - } - } - } - - // Print one line per stream that has a new sample. - for (size_t i = 0; i < stream_count; ++i) + for (size_t i = 0; i < trackers.size(); ++i) { - const auto& tracked = tracker->get_stream_data(*session, i); + const auto& tracked = trackers[i]->get_data(*session); if (!tracked.data || (last_sequences[i].has_value() && tracked.data->sequence_number == last_sequences[i].value())) { diff --git a/src/core/deviceio_base/cpp/inc/deviceio_base/frame_metadata_tracker_oak_base.hpp b/src/core/deviceio_base/cpp/inc/deviceio_base/frame_metadata_tracker_oak_base.hpp index cebeb7c13..3add0a86c 100644 --- a/src/core/deviceio_base/cpp/inc/deviceio_base/frame_metadata_tracker_oak_base.hpp +++ b/src/core/deviceio_base/cpp/inc/deviceio_base/frame_metadata_tracker_oak_base.hpp @@ -5,8 +5,6 @@ #include "tracker.hpp" -#include - namespace core { @@ -16,7 +14,7 @@ struct FrameMetadataOakTrackedT; class IFrameMetadataTrackerOakImpl : public ITrackerImpl { public: - virtual const FrameMetadataOakTrackedT& get_stream_data(size_t stream_index) const = 0; + virtual const FrameMetadataOakTrackedT& get_data() const = 0; }; } // namespace core diff --git a/src/core/deviceio_trackers/cpp/frame_metadata_tracker_oak.cpp b/src/core/deviceio_trackers/cpp/frame_metadata_tracker_oak.cpp index 5c7485448..c48561441 100644 --- a/src/core/deviceio_trackers/cpp/frame_metadata_tracker_oak.cpp +++ b/src/core/deviceio_trackers/cpp/frame_metadata_tracker_oak.cpp @@ -3,7 +3,6 @@ #include "inc/deviceio_trackers/frame_metadata_tracker_oak.hpp" -#include #include namespace core @@ -13,32 +12,14 @@ namespace core // FrameMetadataTrackerOak // ============================================================================ -FrameMetadataTrackerOak::FrameMetadataTrackerOak(const std::string& collection_prefix, - const std::vector& streams, - size_t max_flatbuffer_size) - : collection_prefix_(collection_prefix), streams_(streams), max_flatbuffer_size_(max_flatbuffer_size) +FrameMetadataTrackerOak::FrameMetadataTrackerOak(const std::string& collection_id, size_t max_flatbuffer_size) + : collection_id_(collection_id), max_flatbuffer_size_(max_flatbuffer_size) { - if (streams.empty()) - { - throw std::runtime_error("FrameMetadataTrackerOak: at least one stream is required"); - } - - for (auto type : streams) - { - const char* name = EnumNameStreamType(type); - if (name == nullptr) - { - throw std::invalid_argument("FrameMetadataTrackerOak: invalid StreamType value " + - std::to_string(static_cast(type))); - } - m_stream_names.emplace_back(name); - } } -const FrameMetadataOakTrackedT& FrameMetadataTrackerOak::get_stream_data(const ITrackerSession& session, - size_t stream_index) const +const FrameMetadataOakTrackedT& FrameMetadataTrackerOak::get_data(const ITrackerSession& session) const { - return static_cast(session.get_tracker_impl(*this)).get_stream_data(stream_index); + return static_cast(session.get_tracker_impl(*this)).get_data(); } } // namespace core diff --git a/src/core/deviceio_trackers/cpp/inc/deviceio_trackers/frame_metadata_tracker_oak.hpp b/src/core/deviceio_trackers/cpp/inc/deviceio_trackers/frame_metadata_tracker_oak.hpp index 105c81cbb..210d1dc03 100644 --- a/src/core/deviceio_trackers/cpp/inc/deviceio_trackers/frame_metadata_tracker_oak.hpp +++ b/src/core/deviceio_trackers/cpp/inc/deviceio_trackers/frame_metadata_tracker_oak.hpp @@ -4,48 +4,43 @@ #pragma once #include -#include #include #include -#include namespace core { /*! - * @brief Multi-stream tracker for OAK FrameMetadataOak. + * @brief Single-stream tracker for OAK FrameMetadataOak. * - * Maintains one SchemaTracker per stream. Each stream is identified by its - * StreamType enum name (e.g., "Color", "MonoLeft"). + * Identified by a single collection_id (e.g., "oak_camera/Color"). + * To track multiple streams, create one tracker instance per stream. * * Usage: * @code - * auto tracker = std::make_shared( - * "oak_camera", {StreamType_Color, StreamType_MonoLeft}); - * // ... create session with tracker ... + * auto color_tracker = std::make_shared("oak_camera/Color"); + * auto mono_tracker = std::make_shared("oak_camera/MonoLeft"); + * // ... create session with both trackers ... * session->update(); - * const auto& color = tracker->get_stream_data(*session, 0); + * const auto& color = color_tracker->get_data(*session); * if (color.data) - * std::cout << EnumNameStreamType(color.data->stream) << " seq=" << color.data->sequence_number << std::endl; + * std::cout << color.data->sequence_number << std::endl; * @endcode */ class FrameMetadataTrackerOak : public ITracker { public: - //! Default maximum FlatBuffer size for individual FrameMetadataOak messages. + //! Default maximum FlatBuffer size for FrameMetadataOak messages. static constexpr size_t DEFAULT_MAX_FLATBUFFER_SIZE = 128; /*! - * @brief Constructs a multi-stream FrameMetadataOak tracker. - * @param collection_prefix Base prefix for per-stream collection IDs. - * Each stream gets collection_id = "{collection_prefix}/{StreamName}". - * @param streams Stream types to track. - * @param max_flatbuffer_size Maximum serialized FlatBuffer size per stream (default: 128 bytes). + * @brief Constructs a FrameMetadataTrackerOak. + * @param collection_id Logical stream identifier matching the data source (e.g., "oak_camera/Color"). + * @param max_flatbuffer_size Maximum serialized FlatBuffer size (default: 128 bytes). */ - FrameMetadataTrackerOak(const std::string& collection_prefix, - const std::vector& streams, - size_t max_flatbuffer_size = DEFAULT_MAX_FLATBUFFER_SIZE); + explicit FrameMetadataTrackerOak(const std::string& collection_id, + size_t max_flatbuffer_size = DEFAULT_MAX_FLATBUFFER_SIZE); std::string_view get_name() const override { @@ -53,30 +48,16 @@ class FrameMetadataTrackerOak : public ITracker } /*! - * @brief Get per-stream frame metadata. + * @brief Get frame metadata for the tracked stream. * @param session Active ITrackerSession. - * @param stream_index Index into the streams vector passed at construction. - * @return Reference to the FrameMetadataOakTrackedT for that stream. + * @return Reference to the FrameMetadataOakTrackedT. * The inner @c data pointer is null until the first frame arrives. - * When @c data is non-null, nested fields in FrameMetadataOakT are - * safe to read. */ - const FrameMetadataOakTrackedT& get_stream_data(const ITrackerSession& session, size_t stream_index) const; + const FrameMetadataOakTrackedT& get_data(const ITrackerSession& session) const; - //! Number of streams this tracker is configured for. - size_t get_stream_count() const + const std::string& collection_id() const { - return m_stream_names.size(); - } - - const std::string& collection_prefix() const - { - return collection_prefix_; - } - - const std::vector& streams() const - { - return streams_; + return collection_id_; } size_t max_flatbuffer_size() const @@ -84,18 +65,11 @@ class FrameMetadataTrackerOak : public ITracker return max_flatbuffer_size_; } - const std::vector& get_stream_names() const - { - return m_stream_names; - } - private: static constexpr const char* TRACKER_NAME = "FrameMetadataTrackerOak"; - std::string collection_prefix_; - std::vector streams_; + std::string collection_id_; size_t max_flatbuffer_size_{ DEFAULT_MAX_FLATBUFFER_SIZE }; - std::vector m_stream_names; }; } // namespace core diff --git a/src/core/deviceio_trackers/python/tracker_bindings.cpp b/src/core/deviceio_trackers/python/tracker_bindings.cpp index a36b99213..c3edaa384 100644 --- a/src/core/deviceio_trackers/python/tracker_bindings.cpp +++ b/src/core/deviceio_trackers/python/tracker_bindings.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -168,19 +169,17 @@ PYBIND11_MODULE(_deviceio_trackers, m) py::class_>( m, "FrameMetadataTrackerOak") - .def(py::init&, size_t>(), py::arg("collection_prefix"), - py::arg("streams"), + .def(py::init(), py::arg("collection_id"), py::arg("max_flatbuffer_size") = core::FrameMetadataTrackerOak::DEFAULT_MAX_FLATBUFFER_SIZE, - "Construct a multi-stream FrameMetadataTrackerOak") + "Construct a FrameMetadataTrackerOak for the given collection_id (e.g. \"oak_camera/Color\"); " + "create one instance per stream") .def( - "get_stream_data", - [](const core::FrameMetadataTrackerOak& self, const core::ITrackerSession& session, size_t stream_index) - { return share_tracked(self.get_stream_data(session, stream_index)); }, - py::arg("session"), py::arg("stream_index"), - "Get FrameMetadataOakTrackedT for a specific stream by index; .data is None until first frame " - "arrives" TRACKED_LIFETIME_DOC) - .def_property_readonly("stream_count", &core::FrameMetadataTrackerOak::get_stream_count, - "Number of streams this tracker is configured for"); + "get_data", + [](const core::FrameMetadataTrackerOak& self, const core::ITrackerSession& session) + { return share_tracked(self.get_data(session)); }, + py::arg("session"), + "Get FrameMetadataOakTrackedT for the tracked stream; .data is None until first frame " + "arrives" TRACKED_LIFETIME_DOC); py::class_>( m, "Generic3AxisPedalTracker") diff --git a/src/core/live_trackers/cpp/live_deviceio_factory.cpp b/src/core/live_trackers/cpp/live_deviceio_factory.cpp index 940e7b879..b78cab73f 100644 --- a/src/core/live_trackers/cpp/live_deviceio_factory.cpp +++ b/src/core/live_trackers/cpp/live_deviceio_factory.cpp @@ -552,7 +552,7 @@ std::unique_ptr LiveDeviceIOFactory::create_frame_ std::unique_ptr channels; if (should_record(tracker)) { - channels = LiveFrameMetadataTrackerOakImpl::create_mcap_channels(*writer_, get_name(tracker), tracker); + channels = LiveFrameMetadataTrackerOakImpl::create_mcap_channels(*writer_, get_name(tracker)); } return std::make_unique(handles_, tracker, std::move(channels)); } diff --git a/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.cpp b/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.cpp index fa28deb80..ef3a633a6 100644 --- a/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.cpp +++ b/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.cpp @@ -6,8 +6,8 @@ #include #include -#include #include +#include namespace core { @@ -15,21 +15,14 @@ namespace core namespace { -std::vector make_oak_tensor_configs(const FrameMetadataTrackerOak* tracker) +SchemaTrackerConfig make_oak_tensor_config(const FrameMetadataTrackerOak* tracker) { - std::vector configs; - configs.reserve(tracker->streams().size()); - for (auto type : tracker->streams()) - { - const char* name = EnumNameStreamType(type); - SchemaTrackerConfig cfg; - cfg.collection_id = tracker->collection_prefix() + "/" + name; - cfg.max_flatbuffer_size = tracker->max_flatbuffer_size(); - cfg.tensor_identifier = "frame_metadata"; - cfg.localized_name = std::string("FrameMetadataTracker_") + name; - configs.push_back(std::move(cfg)); - } - return configs; + SchemaTrackerConfig cfg; + cfg.collection_id = tracker->collection_id(); + cfg.max_flatbuffer_size = tracker->max_flatbuffer_size(); + cfg.tensor_identifier = "frame_metadata"; + cfg.localized_name = "FrameMetadataTrackerOak"; + return cfg; } } // namespace @@ -38,46 +31,30 @@ std::vector make_oak_tensor_configs(const FrameMetadataTrac // LiveFrameMetadataTrackerOakImpl // ============================================================================ -std::unique_ptr LiveFrameMetadataTrackerOakImpl::create_mcap_channels( - mcap::McapWriter& writer, std::string_view base_name, const FrameMetadataTrackerOak* tracker) +std::unique_ptr LiveFrameMetadataTrackerOakImpl::create_mcap_channels(mcap::McapWriter& writer, + std::string_view base_name) { - return std::make_unique( - writer, base_name, OakRecordingTraits::schema_name, tracker->get_stream_names()); + return std::make_unique(writer, base_name, OakRecordingTraits::schema_name, + std::vector(OakRecordingTraits::recording_channels.begin(), + OakRecordingTraits::recording_channels.end())); } LiveFrameMetadataTrackerOakImpl::LiveFrameMetadataTrackerOakImpl(const OpenXRSessionHandles& handles, const FrameMetadataTrackerOak* tracker, std::unique_ptr mcap_channels) - : mcap_channels_(std::move(mcap_channels)) + : mcap_channels_(std::move(mcap_channels)), + reader_(handles, make_oak_tensor_config(tracker), mcap_channels_.get(), /*mcap_channel_index=*/0) { - auto configs = make_oak_tensor_configs(tracker); - for (size_t i = 0; i < configs.size(); ++i) - { - StreamState state; - state.reader = std::make_unique(handles, std::move(configs[i]), mcap_channels_.get(), i); - m_streams.push_back(std::move(state)); - } } void LiveFrameMetadataTrackerOakImpl::update(int64_t /*monotonic_time_ns*/) { - // Policy: per-stream SchemaTracker throws on critical OpenXR/tensor API failures. - // Missing stream collection/no fresh sample are treated as common non-fatal cases. - for (auto& stream : m_streams) - { - stream.reader->update(stream.tracked.data); - } + reader_.update(tracked_.data); } -const FrameMetadataOakTrackedT& LiveFrameMetadataTrackerOakImpl::get_stream_data(size_t stream_index) const +const FrameMetadataOakTrackedT& LiveFrameMetadataTrackerOakImpl::get_data() const { - if (stream_index >= m_streams.size()) - { - throw std::runtime_error("FrameMetadataTrackerOak::get_stream_data: invalid stream_index " + - std::to_string(stream_index) + " (have " + std::to_string(m_streams.size()) + - " streams)"); - } - return m_streams[stream_index].tracked; + return tracked_; } } // namespace core diff --git a/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.hpp b/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.hpp index f839eca7b..a055a1c96 100644 --- a/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.hpp +++ b/src/core/live_trackers/cpp/live_frame_metadata_tracker_oak_impl.hpp @@ -13,7 +13,6 @@ #include #include #include -#include namespace core { @@ -28,9 +27,7 @@ class LiveFrameMetadataTrackerOakImpl : public IFrameMetadataTrackerOakImpl { return SchemaTrackerBase::get_required_extensions(); } - static std::unique_ptr create_mcap_channels(mcap::McapWriter& writer, - std::string_view base_name, - const FrameMetadataTrackerOak* tracker); + static std::unique_ptr create_mcap_channels(mcap::McapWriter& writer, std::string_view base_name); LiveFrameMetadataTrackerOakImpl(const OpenXRSessionHandles& handles, const FrameMetadataTrackerOak* tracker, @@ -42,17 +39,12 @@ class LiveFrameMetadataTrackerOakImpl : public IFrameMetadataTrackerOakImpl LiveFrameMetadataTrackerOakImpl& operator=(LiveFrameMetadataTrackerOakImpl&&) = delete; void update(int64_t monotonic_time_ns) override; - const FrameMetadataOakTrackedT& get_stream_data(size_t stream_index) const override; + const FrameMetadataOakTrackedT& get_data() const override; private: - struct StreamState - { - std::unique_ptr reader; - FrameMetadataOakTrackedT tracked; - }; - std::unique_ptr mcap_channels_; - std::vector m_streams; + OakSchemaTracker reader_; + FrameMetadataOakTrackedT tracked_; }; } // namespace core diff --git a/src/core/mcap/cpp/inc/mcap/recording_traits.hpp b/src/core/mcap/cpp/inc/mcap/recording_traits.hpp index 4e5b61b4a..37e0c7175 100644 --- a/src/core/mcap/cpp/inc/mcap/recording_traits.hpp +++ b/src/core/mcap/cpp/inc/mcap/recording_traits.hpp @@ -82,6 +82,7 @@ struct Se3TrackerRecordingTraits struct OakRecordingTraits { static constexpr std::string_view schema_name = "core.FrameMetadataOakRecord"; + static constexpr std::array recording_channels = { "frame_metadata" }; }; struct MessageChannelRecordingTraits diff --git a/src/plugins/oak/README.md b/src/plugins/oak/README.md index 289ba7362..ef7a83236 100644 --- a/src/plugins/oak/README.md +++ b/src/plugins/oak/README.md @@ -90,32 +90,35 @@ controllers, etc.): 1. **Plugin** — launch `oak_camera` via PluginManager or `TeleopSession`'s `PluginConfig`, passing `--collection-prefix` so the plugin pushes metadata via OpenXR `SchemaPusher`. -2. **Host tracker** — create `FrameMetadataTrackerOak` with the **same** - collection prefix and stream list. TeleopSession's DeviceIO layer uses the - live tracker implementation to read the pushed tensors and write MCAP - channels. +2. **Host trackers** — create one `FrameMetadataTrackerOak` per stream, each + with a `collection_id` matching `{collection_prefix}/{StreamName}`. + TeleopSession's DeviceIO layer uses the live tracker implementation to read + the pushed tensors and write MCAP channels. 3. **MCAP config** — add the tracker to both `TeleopSessionConfig.trackers` and `McapRecordingConfig.tracker_names`. ```python from pathlib import Path -from isaacteleop.deviceio import FrameMetadataTrackerOak, McapRecordingConfig, StreamType +from isaacteleop.deviceio import FrameMetadataTrackerOak, McapRecordingConfig from isaacteleop.teleop_session_manager import PluginConfig, TeleopSession, TeleopSessionConfig PLUGIN_ROOT = Path("build/src/plugins") # or your installed plugin search path COLLECTION_PREFIX = "oak_camera" -STREAMS = [StreamType.Color, StreamType.MonoLeft] +STREAM_NAMES = ["Color", "MonoLeft"] -oak_tracker = FrameMetadataTrackerOak(COLLECTION_PREFIX, STREAMS) +oak_trackers = [ + FrameMetadataTrackerOak(f"{COLLECTION_PREFIX}/{name}") + for name in STREAM_NAMES +] config = TeleopSessionConfig( app_name="OakTeleop", pipeline=pipeline, # your retargeting pipeline - trackers=[oak_tracker], + trackers=oak_trackers, mcap_config=McapRecordingConfig( "recording.mcap", - [(oak_tracker, "oak_metadata")], + [(t, f"oak_metadata/{name}") for t, name in zip(oak_trackers, STREAM_NAMES)], ), plugins=[ PluginConfig(