diff --git a/docs/articles_en/assets/snippets/gpu/remote_objects_creation.cpp b/docs/articles_en/assets/snippets/gpu/remote_objects_creation.cpp index b62af47e3490c6..eb7fbc36b6f7b1 100644 --- a/docs/articles_en/assets/snippets/gpu/remote_objects_creation.cpp +++ b/docs/articles_en/assets/snippets/gpu/remote_objects_creation.cpp @@ -69,6 +69,17 @@ int main() { //! [wrap_cpu_pointer] } +{ + //! [wrap_file] + // The plugin memory-maps the file and keeps the mapping alive for the tensor lifetime, + // so the file must not be modified until the returned tensor is destroyed. + ov::intel_gpu::FileDescriptor file_descriptor{"input.bin", + /*offset_in_bytes=*/0, + ov::intel_gpu::FileAccess::READ}; + auto remote_tensor = gpu_context.create_tensor(in_element_type, in_shape, file_descriptor); + //! [wrap_file] +} + { //! [wrap_cl_mem] cl_mem shared_buffer = allocate_cl_mem(input_size); diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device/remote-tensor-api-gpu-plugin.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device/remote-tensor-api-gpu-plugin.rst index 8a014404459f5f..def794430780e8 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device/remote-tensor-api-gpu-plugin.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device/remote-tensor-api-gpu-plugin.rst @@ -233,6 +233,19 @@ For more details, see the code snippets below: :language: cpp :fragment: [wrap_usm_pointer] + .. tab-item:: CPU pointer + :sync: cpu-pointer + + Use this overload when your application owns a CPU virtual address, for example memory + allocated with ``ov::util::aligned_alloc`` or memory mapped from a file. On the OpenCL + backend, the pointer address and allocation size must be aligned to + ``ov::intel_gpu::cacheline_size``. The memory must remain valid for the whole lifetime of + the created remote tensor. + + .. doxygensnippet:: docs/articles_en/assets/snippets/gpu/remote_objects_creation.cpp + :language: cpp + :fragment: [wrap_cpu_pointer] + .. tab-item:: cl_mem :sync: cl-mem @@ -267,6 +280,20 @@ For more details, see the code snippets below: The ``shape`` and ``element type`` must describe the same memory layout as the external buffer. The handle must remain valid for the whole lifetime of the created remote tensor. + .. tab-item:: file + :sync: file + + Use this overload to wrap tensor data stored in a file. The access mode is declared in the + descriptor: ``FileAccess::READ`` maps the file read-only and may only be used as an inference + input, while ``FileAccess::READ_WRITE`` requires a writable file, may be used as an inference + output, and makes changes done through the tensor visible in the file. The plugin keeps the + mapping alive for the whole lifetime of the created remote tensor, so the file must not + otherwise be modified until the tensor is destroyed. + + .. doxygensnippet:: docs/articles_en/assets/snippets/gpu/remote_objects_creation.cpp + :language: cpp + :fragment: [wrap_file] + .. tab-item:: biplanar NV12 surface :sync: biplanar-nv12-surface diff --git a/src/common/util/include/openvino/util/mmap_object.hpp b/src/common/util/include/openvino/util/mmap_object.hpp index 93a5803ffe3baf..f162a56cd6a6fc 100644 --- a/src/common/util/include/openvino/util/mmap_object.hpp +++ b/src/common/util/include/openvino/util/mmap_object.hpp @@ -29,6 +29,19 @@ int64_t get_system_page_size(); */ inline constexpr auto auto_size = std::numeric_limits::max(); +/** + * @brief Access mode of a memory mapping. + */ +enum class MmapMode { + READ, //!< Read-only mapping. + READ_WRITE //!< Read-write mapping, modifications are written back to the file. +}; + +/** + * @brief Id reported by mappings whose content is mutable, so they must never be substituted for one another. + */ +inline constexpr uint64_t no_mapping_id = 0; + /** * @brief This class represents a mapped memory. * Instead of reading files, we can map the memory via mmap for Linux or MapViewOfFile for Windows. @@ -73,12 +86,15 @@ class MappedMemory { * @param no_placeholder When true, skip the Windows 10+ placeholder/VEH mechanism and use the legacy * single-call MapViewOfFile path instead. This guarantees a uniform AllocationBase * across the whole mapping, required for NPU zero-copy blob import. On Linux ignored. + * @param mode Access mode of the mapping. A read_write mapping requires the file to be writable and reports + * no_mapping_id, because get_id() marks an immutable data source that consumers may share. * @return MappedMemory shared ptr object which keep mmaped memory and control the lifetime. */ std::shared_ptr load_mmap_object(const std::filesystem::path& path, size_t offset = 0, size_t size = auto_size, - bool no_placeholder = false); + bool no_placeholder = false, + MmapMode mode = MmapMode::READ); /** * @brief Returns mapped memory for a file from provided file handle (cross-platform). diff --git a/src/common/util/src/os/lin/lin_mmap_object.cpp b/src/common/util/src/os/lin/lin_mmap_object.cpp index b1a43bc0596f13..b534e298bbd896 100644 --- a/src/common/util/src/os/lin/lin_mmap_object.cpp +++ b/src/common/util/src/os/lin/lin_mmap_object.cpp @@ -141,18 +141,18 @@ class MapHolder final : public MappedMemory { public: MapHolder() = default; - void set(const std::filesystem::path& path, const size_t offset, const size_t size) { - int mode = O_RDONLY; + void set(const std::filesystem::path& path, const size_t offset, const size_t size, const MmapMode mmap_mode) { + int mode = (mmap_mode == MmapMode::READ_WRITE) ? O_RDWR : O_RDONLY; int fd = open(path.c_str(), mode); if (fd == -1) { throw std::runtime_error("Can not open file " + util::path_to_string(path) + " for mapping. Ensure that file exists and has appropriate permissions."); } - set_from_fd(fd, offset, size); - m_id = util::get_id_for_file(path, offset, size); + set_from_fd(fd, offset, size, mmap_mode); + m_id = (mmap_mode == MmapMode::READ_WRITE) ? no_mapping_id : util::get_id_for_file(path, offset, size); } - void set_from_fd(const int fd, const size_t offset, const size_t size) { + void set_from_fd(const int fd, const size_t offset, const size_t size, const MmapMode mmap_mode = MmapMode::READ) { m_handle = HandleHolder(fd); struct stat sb = {}; @@ -166,17 +166,21 @@ class MapHolder final : public MappedMemory { } if (m_size > 0) { + const auto prot = (mmap_mode == MmapMode::READ_WRITE) ? (PROT_READ | PROT_WRITE) : PROT_READ; const auto& [aligned_offset, length, gap] = util::make_mmap_region(offset, m_size); m_mapped_view_size = length; - m_mapped_view = mmap(nullptr, length, PROT_READ, MAP_SHARED, fd, aligned_offset); + m_mapped_view = mmap(nullptr, length, prot, MAP_SHARED, fd, aligned_offset); if (m_mapped_view == MAP_FAILED) { throw std::runtime_error("Can not create file mapping for " + std::to_string(fd) + ", err=" + std::strerror(errno)); } m_data = static_cast(m_mapped_view) + gap; } - m_id = - util::u64_hash_combine(static_cast(sb.st_ino), {static_cast(sb.st_dev), offset, size}); + // A read-write mapping is not an immutable data source, so it must not be shared through id-based caches. + m_id = (mmap_mode == MmapMode::READ_WRITE) + ? no_mapping_id + : util::u64_hash_combine(static_cast(sb.st_ino), + {static_cast(sb.st_dev), offset, size}); } uint64_t get_id() const noexcept override { @@ -231,9 +235,10 @@ class MapHolder final : public MappedMemory { std::shared_ptr load_mmap_object(const std::filesystem::path& path, size_t offset, size_t size, - bool /* no_placeholder */) { + bool /* no_placeholder */, + MmapMode mode) { auto holder = std::make_shared(); - holder->set(path, offset, size); + holder->set(path, offset, size, mode); return holder; } diff --git a/src/common/util/src/os/win/win_mmap_object.cpp b/src/common/util/src/os/win/win_mmap_object.cpp index c78fd40c24da05..5043dd62fec4ba 100644 --- a/src/common/util/src/os/win/win_mmap_object.cpp +++ b/src/common/util/src/os/win/win_mmap_object.cpp @@ -288,7 +288,11 @@ class MapHolder : public ov::MappedMemory { MapHolder() = default; ~MapHolder() override; - void set(const std::filesystem::path& path, size_t offset, size_t size, bool no_placeholder = false); + void set(const std::filesystem::path& path, + size_t offset, + size_t size, + bool no_placeholder = false, + MmapMode mode = MmapMode::READ); void set_from_handle(FileHandle handle, size_t offset, size_t size); bool try_remap_slot(uintptr_t fault_addr); @@ -334,7 +338,7 @@ class MapHolder : public ov::MappedMemory { void set_id(HANDLE h, size_t offset, size_t size); /** @brief Core setup shared by set() and set_from_handle(). */ - void setup(HANDLE file_handle, size_t offset, size_t size, bool no_placeholder); + void setup(HANDLE file_handle, size_t offset, size_t size, bool no_placeholder, MmapMode mode); /** @brief Try to establish the placeholder mapping. * Returns true on success; caller falls back to legacy path on false. @@ -342,7 +346,7 @@ class MapHolder : public ov::MappedMemory { bool try_placeholder_setup(size_t aligned_offset, size_t head_pad, size_t total_va_size, size_t file_size); /** @brief Legacy single-call MapViewOfFile path (no partial-release support). */ - void legacy_setup(size_t aligned_offset, size_t head_pad, size_t size); + void legacy_setup(size_t aligned_offset, size_t head_pad, size_t size, MmapMode mode); /** * @brief Computes the clamped, gran-aligned VA range to evict. @@ -614,9 +618,10 @@ bool MapHolder::try_placeholder_setup(size_t aligned_offset, size_t head_pad, si return true; } -void MapHolder::legacy_setup(size_t aligned_offset, size_t head_pad, size_t size) { +void MapHolder::legacy_setup(size_t aligned_offset, size_t head_pad, size_t size, MmapMode mode) { + const DWORD access = (mode == MmapMode::READ_WRITE) ? FILE_MAP_ALL_ACCESS : FILE_MAP_READ; if (auto view = ::MapViewOfFile(m_handle.get(), - FILE_MAP_READ, + access, static_cast(aligned_offset >> 32), static_cast(aligned_offset & 0xFFFFFFFF), head_pad + size)) { @@ -627,7 +632,7 @@ void MapHolder::legacy_setup(size_t aligned_offset, size_t head_pad, size_t size } } -void MapHolder::setup(HANDLE file_handle, size_t offset, size_t size, bool no_placeholder) { +void MapHolder::setup(HANDLE file_handle, size_t offset, size_t size, bool no_placeholder, MmapMode mode) { LARGE_INTEGER file_size_li{}; if (!::GetFileSizeEx(file_handle, &file_size_li)) { throw std::runtime_error{"GetFileSizeEx failed: " + std::to_string(::GetLastError())}; @@ -646,28 +651,35 @@ void MapHolder::setup(HANDLE file_handle, size_t offset, size_t size, bool no_pl const size_t total_va_size = util::align_size_up(r_length, gran); set_id(file_handle, offset, size); + if (mode == MmapMode::READ_WRITE) { + // A read-write mapping is not an immutable data source, so it must not be shared through id-based caches. + m_id = no_mapping_id; + } if (m_size == 0) { return; } - // Create a read-only file-mapping object for the whole file. - m_handle = HandleHolder{::CreateFileMappingW(file_handle, nullptr, PAGE_READONLY, 0, 0, nullptr)}; + const DWORD protect = (mode == MmapMode::READ_WRITE) ? PAGE_READWRITE : PAGE_READONLY; + m_handle = HandleHolder{::CreateFileMappingW(file_handle, nullptr, protect, 0, 0, nullptr)}; if (!m_handle.valid()) { throw std::runtime_error{"CreateFileMappingW failed: " + std::to_string(::GetLastError())}; } // When no_placeholder is set, skip the placeholder/VEH path to guarantee a single uniform AllocationBase // (required for NPU zero-copy blob import). Otherwise prefer placeholder for RSS reduction. - if (no_placeholder || !try_placeholder_setup(m_aligned_offset, head_pad, total_va_size, file_size)) { - legacy_setup(m_aligned_offset, head_pad, m_size); + // RW mappings are ignored by the current VEH registration: the handler only remaps read faults. + if (no_placeholder || mode == MmapMode::READ_WRITE || + !try_placeholder_setup(m_aligned_offset, head_pad, total_va_size, file_size)) { + legacy_setup(m_aligned_offset, head_pad, m_size, mode); } } -void MapHolder::set(const std::filesystem::path& path, size_t offset, size_t size, bool no_placeholder) { +void MapHolder::set(const std::filesystem::path& path, size_t offset, size_t size, bool no_placeholder, MmapMode mode) { + const bool writable = mode == MmapMode::READ_WRITE; auto fh = ::CreateFileW(path.c_str(), - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_DELETE, + writable ? (GENERIC_READ | GENERIC_WRITE) : GENERIC_READ, + writable ? (FILE_SHARE_READ | FILE_SHARE_WRITE| FILE_SHARE_DELETE) : (FILE_SHARE_READ | FILE_SHARE_DELETE), nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, @@ -678,7 +690,7 @@ void MapHolder::set(const std::filesystem::path& path, size_t offset, size_t siz } HandleHolder fh_holder{fh}; - setup(fh, offset, size, no_placeholder); + setup(fh, offset, size, no_placeholder, mode); // Keep the file handle alive so the section object can always resolve page faults // back to the original file data, even if the caller deletes or renames the file. // FILE_SHARE_DELETE allows std::filesystem::remove() to succeed while the mapping is alive. @@ -702,7 +714,7 @@ void MapHolder::set_from_handle(FileHandle handle, size_t offset, size_t size) { throw std::runtime_error{"DuplicateHandle failed: " + std::to_string(::GetLastError())}; } HandleHolder owned{dup}; - setup(owned.get(), offset, size, false); + setup(owned.get(), offset, size, false, MmapMode::READ); // owned goes out of scope here: file handle closed. // m_handle (section object) keeps the file data accessible independently. } @@ -947,9 +959,10 @@ void MapHolder::hint_evict(size_t offset, size_t size) noexcept { std::shared_ptr load_mmap_object(const std::filesystem::path& path, size_t offset, size_t size, - bool no_placeholder) { + bool no_placeholder, + MmapMode mode) { auto holder = std::make_shared(); - holder->set(path, offset, size, no_placeholder); + holder->set(path, offset, size, no_placeholder, mode); return holder; } diff --git a/src/core/tests/mmap_object.cpp b/src/core/tests/mmap_object.cpp index 468912428e3758..f89b8d159fc324 100644 --- a/src/core/tests/mmap_object.cpp +++ b/src/core/tests/mmap_object.cpp @@ -240,6 +240,60 @@ INSTANTIATE_TEST_SUITE_P(MappedMemory, ::testing::ValuesIn(std::vector{true, false})), RangedMappingTest::test_name); +class ReadWriteMappingTest : public ::testing::Test { +protected: + std::filesystem::path m_file_path; + std::vector m_content; + static constexpr size_t k_file_size = 128 * 1024; + + void SetUp() override { + m_content = utils::make_modulo_sequence_pattern(k_file_size); + m_file_path = utils::generateTestFilePrefix() + "_rw_mapping"; + ov::util::save_binary(m_file_path, m_content.data(), m_content.size()); + } + + void TearDown() override { + std::filesystem::remove(m_file_path); + } + + std::vector read_file() const { + std::vector data(static_cast(std::filesystem::file_size(m_file_path))); + std::ifstream is(m_file_path, std::ios::binary); + is.read(reinterpret_cast(data.data()), static_cast(data.size())); + return data; + } +}; + +TEST_F(ReadWriteMappingTest, writes_at_offset_leave_other_bytes_intact) { + constexpr size_t k_offset = 64 * 1024; + constexpr size_t k_size = 512; + + auto expected = m_content; + std::fill_n(expected.begin() + k_offset, k_size, uint8_t{0x5A}); + ASSERT_NE(expected, m_content); + + { + auto mm = load_mmap_object(m_file_path, k_offset, k_size, false, MmapMode::READ_WRITE); + std::fill_n(reinterpret_cast(mm->data()), k_size, uint8_t{0x5A}); + } + + EXPECT_THAT(read_file(), ElementsAreArray(expected)); +} + +TEST_F(ReadWriteMappingTest, read_write_mappings_report_no_mapping_id) { + auto rw_whole = load_mmap_object(m_file_path, 0, auto_size, false, MmapMode::READ_WRITE); + auto rw_part = load_mmap_object(m_file_path, 128, 256, false, MmapMode::READ_WRITE); + auto ro = load_mmap_object(m_file_path); + + ASSERT_NE(rw_whole, nullptr); + ASSERT_NE(rw_part, nullptr); + ASSERT_NE(ro, nullptr); + + EXPECT_EQ(rw_whole->get_id(), no_mapping_id); + EXPECT_EQ(rw_part->get_id(), no_mapping_id); + EXPECT_NE(ro->get_id(), no_mapping_id); +} + class HintEvictTest : public ::testing::Test { protected: std::filesystem::path m_file_path; diff --git a/src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp b/src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp index 82dbc0f5c977f7..4f99b035b345ce 100644 --- a/src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp +++ b/src/inference/include/openvino/runtime/intel_gpu/ocl/ocl.hpp @@ -10,7 +10,6 @@ */ #pragma once -#include #include #include "openvino/runtime/core.hpp" @@ -43,6 +42,7 @@ using gpu_handle_param = void*; using SharedBufferHandle = ov::intel_gpu::SharedBufferHandle; using VirtualAddressMemory = ov::intel_gpu::VirtualAddressMemory; +using FileDescriptor = ov::intel_gpu::FileDescriptor; /** * @brief This class represents an abstraction for GPU plugin remote tensor @@ -346,6 +346,25 @@ class ClContext : public RemoteContext { return create_tensor(type, shape, params).as(); } + /** + * @brief This function is used to obtain a remote tensor object from a file. + * The plugin memory-maps the file and keeps the mapping alive for the whole tensor lifetime, + * so the file must not be modified until the returned tensor is destroyed. + * @param type Tensor element type + * @param shape Tensor shape + * @param file_descriptor Descriptor with the path, offset and access mode of the file containing tensor data. + * The offset must be a multiple of the system memory mapping alignment: the page size on Linux + * (typically 4 KiB) and the allocation granularity on Windows (typically 64 KiB). + * FileAccess::READ_WRITE additionally requires the file to be writable by the calling process + * and makes the tensor writes visible in the file. + * @return A remote tensor instance + */ + ClBufferTensor create_tensor(const element::Type type, const Shape& shape, const FileDescriptor& file_descriptor) { + AnyMap params = {{ov::intel_gpu::shared_mem_type.name(), ov::intel_gpu::SharedMemType::MMAPED_FILE}, + {ov::intel_gpu::file_descriptor.name(), file_descriptor}}; + return create_tensor(type, shape, params).as(); + } + /** * @brief This function is used to obtain remote tensor object from user-supplied USM pointer * @param type Tensor element type diff --git a/src/inference/include/openvino/runtime/intel_gpu/remote_properties.hpp b/src/inference/include/openvino/runtime/intel_gpu/remote_properties.hpp index c6cd21d1140236..048d02845769d5 100644 --- a/src/inference/include/openvino/runtime/intel_gpu/remote_properties.hpp +++ b/src/inference/include/openvino/runtime/intel_gpu/remote_properties.hpp @@ -10,6 +10,8 @@ */ #pragma once +#include + #include "openvino/runtime/properties.hpp" namespace ov { @@ -119,6 +121,7 @@ enum class SharedMemType { BUFFER_FROM_HANDLE = 7, //!< OS-level external memory handle (e.g. DX12 NT handle on Windows, //!< DMA-BUF fd on Linux) imported by the plugin into a cl_mem CPU_VA = 8, //!< Shared mmap-backed/aligned allocated host pointer mapped by plugin + MMAPED_FILE = 9, //!< Memory-mapped file buffer read and wrapped by the plugin }; /** @@ -151,6 +154,8 @@ inline std::ostream& operator<<(std::ostream& os, const SharedMemType& share_mem return os << "DX_BUFFER"; case SharedMemType::BUFFER_FROM_HANDLE: return os << "BUFFER_FROM_HANDLE"; + case SharedMemType::MMAPED_FILE: + return os << "MMAPED_FILE"; default: OPENVINO_THROW("Unsupported memory type"); } @@ -177,6 +182,8 @@ inline std::istream& operator>>(std::istream& is, SharedMemType& share_mem_type) share_mem_type = SharedMemType::DX_BUFFER; } else if (str == "BUFFER_FROM_HANDLE") { share_mem_type = SharedMemType::BUFFER_FROM_HANDLE; + } else if (str == "MMAPED_FILE") { + share_mem_type = SharedMemType::MMAPED_FILE; } else { OPENVINO_THROW("Unsupported memory type: ", str); } @@ -260,5 +267,42 @@ struct VirtualAddressMemory { void* ptr = nullptr; int64_t size = -1; ///< Buffer size in bytes; -1 means "derive from tensor shape" }; + +/** + * @brief Enum to define how a memory-mapped file is accessed by the plugin + * @ingroup ov_runtime_ocl_gpu_cpp_api + */ +enum class FileAccess { + READ = 0, //!< Tensor data is only read from the file + READ_WRITE = 1 //!< Tensor data is also written back to the file; requires a writable file +}; + +/** + * @brief File descriptor for wrapping tensor data memory-mapped from a file as a GPU plugin tensor. + * The plugin memory-maps the file and keeps the mapping alive for the whole tensor lifetime, + * so the file must not be modified until the returned tensor is destroyed. + * @ingroup ov_runtime_ocl_gpu_cpp_api + */ +struct FileDescriptor { // need to be merged with ov::intel_npu::FileDescriptor in future + explicit FileDescriptor(const std::filesystem::path& file_path, + std::size_t offset_in_bytes = 0, + FileAccess file_access = FileAccess::READ) + : path(file_path), + offset(offset_in_bytes), + access(file_access) { + OPENVINO_ASSERT(!file_path.empty(), "[GPU] Provided file path is empty."); + } + + std::filesystem::path path; ///< File path + std::size_t offset = 0; ///< Offset in bytes to read from the file + FileAccess access = FileAccess::READ; ///< Access mode of the mapping +}; + +/** + * @brief This key identifies the file descriptor + * in a memory-mapped tensor parameter map. + * @ingroup ov_runtime_ocl_gpu_cpp_api + */ +static constexpr Property file_descriptor{"FILE_DESCRIPTOR"}; } // namespace intel_gpu } // namespace ov diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_context.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_context.hpp index 449db2e850bf98..1a1b9cf3403a98 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_context.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_context.hpp @@ -18,6 +18,7 @@ #include "intel_gpu/plugin/common_utils.hpp" #include +#include #include #include @@ -85,6 +86,7 @@ class RemoteContextImpl : public ov::IRemoteContext { std::shared_ptr reuse_memory(const ov::element::Type type, const ov::Shape& shape, cldnn::shared_handle mem, TensorType tensor_type); std::shared_ptr reuse_memory_from_cpu_va(const ov::element::Type type, const ov::Shape& shape, VirtualAddressMemory cpu_va, TensorType tensor_type); std::shared_ptr reuse_memory_from_handle(const ov::element::Type type, const ov::Shape& shape, SharedBufferHandle handle, TensorType tensor_type); + std::shared_ptr reuse_memory_from_file(const ov::element::Type type, const ov::Shape& shape, const std::filesystem::path& file_path, size_t offset, ov::intel_gpu::FileAccess access); std::shared_ptr create_buffer(const ov::element::Type type, const ov::Shape& shape); std::shared_ptr create_usm(const ov::element::Type type, const ov::Shape& shape, TensorType alloc_type); void check_if_shared() const; diff --git a/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_tensor.hpp b/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_tensor.hpp index 0b1378dc87f5d1..831e48980163fb 100644 --- a/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_tensor.hpp +++ b/src/plugins/intel_gpu/include/intel_gpu/plugin/remote_tensor.hpp @@ -19,6 +19,8 @@ #endif #include "openvino/runtime/iremote_tensor.hpp" #include "openvino/runtime/intel_gpu/remote_properties.hpp" +#include "openvino/runtime/tensor.hpp" +#include "openvino/util/mmap_object.hpp" #include "intel_gpu/runtime/memory_caps.hpp" #include "intel_gpu/runtime/memory.hpp" @@ -43,7 +45,9 @@ class RemoteTensorImpl : public ov::IRemoteTensor { cldnn::shared_surface surf = 0, uint32_t plane = 0, ov::intel_gpu::SharedBufferHandle shared_buffer_handle = {}, - ov::intel_gpu::VirtualAddressMemory va_mem = ov::intel_gpu::VirtualAddressMemory(nullptr)); + ov::intel_gpu::VirtualAddressMemory va_mem = ov::intel_gpu::VirtualAddressMemory(nullptr), + std::shared_ptr mapped_memory = nullptr, + bool mapped_memory_read_only = true); ~RemoteTensorImpl() override; const AnyMap& get_properties() const override; @@ -88,6 +92,8 @@ class RemoteTensorImpl : public ov::IRemoteTensor { uint32_t m_plane; ov::intel_gpu::SharedBufferHandle m_shared_buffer_handle; ov::intel_gpu::VirtualAddressMemory m_va_mem; + std::shared_ptr m_mapped_memory; // keeps the file mapping alive for the whole tensor lifetime + bool m_mapped_memory_read_only = true; size_t m_hash = 0; bool supports_caching() const; diff --git a/src/plugins/intel_gpu/src/plugin/remote_context.cpp b/src/plugins/intel_gpu/src/plugin/remote_context.cpp index d09bc5bccc1e0e..fdf2e9f1044151 100644 --- a/src/plugins/intel_gpu/src/plugin/remote_context.cpp +++ b/src/plugins/intel_gpu/src/plugin/remote_context.cpp @@ -4,13 +4,24 @@ #include "openvino/runtime/intel_gpu/remote_properties.hpp" #include "openvino/runtime/make_tensor.hpp" +#include "openvino/runtime/tensor.hpp" +#include "openvino/core/memory_util.hpp" +#include "openvino/util/mmap_object.hpp" #include "intel_gpu/plugin/remote_context.hpp" #include "intel_gpu/plugin/remote_tensor.hpp" #include "intel_gpu/plugin/usm_host_tensor.hpp" #include "intel_gpu/runtime/itt.hpp" #include "intel_gpu/runtime/device_query.hpp" +#include "intel_gpu/runtime/utils.hpp" #include +#ifdef _WIN32 +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +#endif + namespace ov::intel_gpu { namespace { @@ -23,6 +34,17 @@ Type extract_object(const ov::AnyMap& params, const ov::Property& p) { return res.as(); } +// Alignment required for a memory mapping offset: allocation granularity on Windows, page size elsewhere. +size_t get_mmap_offset_alignment() { +#ifdef _WIN32 + SYSTEM_INFO sys_info; + GetSystemInfo(&sys_info); + return static_cast(sys_info.dwAllocationGranularity); +#else + return static_cast(ov::util::get_system_page_size()); +#endif +} + ContextType get_default_context_type() { #ifdef OV_GPU_WITH_ZE_RT return ContextType::ZE; @@ -173,6 +195,7 @@ ov::SoPtr RemoteContextImpl::create_tensor(const ov::element: if (ov::intel_gpu::SharedMemType::USM_DEVICE_BUFFER == mem_type) { return { create_usm(type, shape, TensorType::BT_USM_DEVICE_INTERNAL), nullptr }; } + TensorType tensor_type; cldnn::shared_handle mem = nullptr; @@ -186,26 +209,29 @@ ov::SoPtr RemoteContextImpl::create_tensor(const ov::element: tensor_type = TensorType::BT_CPU_VA; mem = extract_object(params, ov::intel_gpu::cpu_va); auto size = extract_object(params, ov::intel_gpu::cpu_va_size); - return {reuse_memory_from_cpu_va(type, shape, VirtualAddressMemory{mem, size}, tensor_type), nullptr}; + return { reuse_memory_from_cpu_va(type, shape, VirtualAddressMemory{mem, size}, tensor_type), nullptr }; + } else if (ov::intel_gpu::SharedMemType::MMAPED_FILE == mem_type) { + const auto fd = extract_object(params, ov::intel_gpu::file_descriptor); + return { reuse_memory_from_file(type, shape, fd.path, fd.offset, fd.access), nullptr }; } else if (ov::intel_gpu::SharedMemType::OCL_IMAGE2D == mem_type) { tensor_type = TensorType::BT_IMG_SHARED; mem = extract_object(params, ov::intel_gpu::mem_handle); #ifdef _WIN32 - } else if (ov::intel_gpu::SharedMemType::DX_BUFFER == mem_type) { - tensor_type = TensorType::BT_DX_BUF_SHARED; - mem = extract_object(params, ov::intel_gpu::dev_object_handle); - check_if_shared(); + } else if (ov::intel_gpu::SharedMemType::DX_BUFFER == mem_type) { + tensor_type = TensorType::BT_DX_BUF_SHARED; + mem = extract_object(params, ov::intel_gpu::dev_object_handle); + check_if_shared(); #endif - } else if (ov::intel_gpu::SharedMemType::BUFFER_FROM_HANDLE == mem_type) { - tensor_type = TensorType::BT_BUF_SHARED_FROM_HANDLE; - const auto os_handle = extract_object(params, ov::intel_gpu::os_handle); - SharedBufferHandle handle{os_handle}; - return { reuse_memory_from_handle(type, shape, handle, tensor_type), nullptr }; - } else { - OPENVINO_THROW("[GPU] Unsupported shared object type ", mem_type); - } + } else if (ov::intel_gpu::SharedMemType::BUFFER_FROM_HANDLE == mem_type) { + tensor_type = TensorType::BT_BUF_SHARED_FROM_HANDLE; + const auto os_handle = extract_object(params, ov::intel_gpu::os_handle); + SharedBufferHandle handle{os_handle}; + return { reuse_memory_from_handle(type, shape, handle, tensor_type), nullptr }; + } else { + OPENVINO_THROW("[GPU] Unsupported shared object type ", mem_type); + } - return { reuse_memory(type, shape, mem, tensor_type), nullptr }; + return { reuse_memory(type, shape, mem, tensor_type), nullptr }; } // For external contexts we try to match underlying handles with default contexts created by plugin to find device name @@ -271,6 +297,48 @@ std::shared_ptr RemoteContextImpl::reuse_memory_from_handle(c return std::make_shared(get_this_shared_ptr(), shape, type, tensor_type, nullptr, 0, 0, handle); } +std::shared_ptr RemoteContextImpl::reuse_memory_from_file(const ov::element::Type type, + const ov::Shape& shape, + const std::filesystem::path& file_path, + size_t offset, + ov::intel_gpu::FileAccess access) { + const auto byte_size = ov::util::get_memory_size_safe(type, shape); + OPENVINO_ASSERT(byte_size, "[GPU] Cannot calculate memory size for element type ", type, " and shape ", shape); + + const auto alignment = get_mmap_offset_alignment(); + OPENVINO_ASSERT(alignment != 0 && offset % alignment == 0, + "[GPU] Offset ", + offset, + " must be a multiple of ", + alignment); + // Memory-map the file. The mapping is retained inside the RemoteTensorImpl so it stays + // alive for the whole tensor lifetime (GPU wraps the host pointer via CL_MEM_USE_HOST_PTR). + const bool read_only = access == ov::intel_gpu::FileAccess::READ; + auto mapped_memory = ov::load_mmap_object(file_path, + offset, + *byte_size, + /*no_placeholder=*/false, + read_only ? ov::MmapMode::READ : ov::MmapMode::READ_WRITE); + + auto import_size = *byte_size; + const auto cacheline_size = static_cast(get_engine().get_device_info().cacheline_size.value_or(0)); + if (cacheline_size > 0 && alignment % cacheline_size == 0) { + import_size = cldnn::align_to(import_size, cacheline_size); + } + + return std::make_shared(get_this_shared_ptr(), + shape, + type, + TensorType::BT_CPU_VA, + nullptr, + 0, + 0, + ov::intel_gpu::SharedBufferHandle{}, + VirtualAddressMemory{mapped_memory->data(), static_cast(import_size)}, + mapped_memory, + read_only); +} + std::shared_ptr RemoteContextImpl::create_buffer(const ov::element::Type type, const ov::Shape& shape) { return std::make_shared(get_this_shared_ptr(), shape, type, TensorType::BT_BUF_INTERNAL); } diff --git a/src/plugins/intel_gpu/src/plugin/remote_tensor.cpp b/src/plugins/intel_gpu/src/plugin/remote_tensor.cpp index 089720756ff7a7..2519fb0b994b2a 100644 --- a/src/plugins/intel_gpu/src/plugin/remote_tensor.cpp +++ b/src/plugins/intel_gpu/src/plugin/remote_tensor.cpp @@ -172,7 +172,9 @@ RemoteTensorImpl::RemoteTensorImpl(RemoteContextImpl::Ptr context, cldnn::shared_surface surf, uint32_t plane, ov::intel_gpu::SharedBufferHandle shared_buffer_handle, - ov::intel_gpu::VirtualAddressMemory va_mem) + ov::intel_gpu::VirtualAddressMemory va_mem, + std::shared_ptr mapped_memory, + bool mapped_memory_read_only) : m_context(context) , m_element_type(element_type) , m_shape(shape) @@ -182,7 +184,9 @@ RemoteTensorImpl::RemoteTensorImpl(RemoteContextImpl::Ptr context, , m_surf(surf) , m_plane(plane) , m_shared_buffer_handle(shared_buffer_handle) - , m_va_mem(va_mem) { + , m_va_mem(va_mem) + , m_mapped_memory(std::move(mapped_memory)) + , m_mapped_memory_read_only(mapped_memory_read_only) { update_hash(); allocate(); } @@ -407,10 +411,19 @@ void RemoteTensorImpl::allocate() { break; } case TensorType::BT_CPU_VA: { - m_memory_object = engine.create_hostbuffer(m_va_mem.ptr, - m_va_mem.size > -1 ? m_va_mem.size : m_layout.bytes_count(), - cldnn::allocation_type::cl_mem, - m_layout); + const auto buffer_size = m_va_mem.size > -1 ? m_va_mem.size : m_layout.bytes_count(); + if (m_mapped_memory && m_mapped_memory_read_only) { + // The plugin owns a read-only mapping (file-mmap case), so the buffer is imported as read-only. + m_memory_object = engine.create_hostbuffer(static_cast(m_va_mem.ptr), + buffer_size, + cldnn::allocation_type::cl_mem, + m_layout); + } else { + m_memory_object = engine.create_hostbuffer(m_va_mem.ptr, + buffer_size, + cldnn::allocation_type::cl_mem, + m_layout); + } break; } #ifdef _WIN32 @@ -458,7 +471,9 @@ bool RemoteTensorImpl::is_shared() const noexcept { } bool RemoteTensorImpl::supports_caching() const { - return is_shared(); + // Memory mapped by the plugin is released together with this tensor, so the cached memory object + // (created with CL_MEM_USE_HOST_PTR) would outlive the host pointer it wraps. + return is_shared() && !m_mapped_memory; } void RemoteTensorImpl::update_hash() { diff --git a/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp b/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp index 5c50a998bcb64d..646010ef610ec2 100644 --- a/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp +++ b/src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp @@ -248,7 +248,7 @@ memory_ptr ocl_engine::create_hostbuffer(void* cpu_address, } memory_ptr ocl_engine::create_hostbuffer(const void* cpu_address, size_t data_size, allocation_type _allocation_type, const layout output_layout) { - return create_hostbuffer_impl(const_cast(cpu_address), data_size, _allocation_type, output_layout, CL_MEM_READ_ONLY); + return create_hostbuffer_impl(const_cast(cpu_address), data_size, _allocation_type, output_layout, CL_MEM_READ_ONLY | CL_MEM_HOST_READ_ONLY); } memory::ptr ocl_engine::reinterpret_buffer(const memory& memory, const layout& new_layout) { diff --git a/src/plugins/intel_gpu/tests/functional/remote_tensor_tests/ocl_remote_tensor_tests.cpp b/src/plugins/intel_gpu/tests/functional/remote_tensor_tests/ocl_remote_tensor_tests.cpp index ff3f6f7e3d0d01..f9e3db17eaa1fd 100644 --- a/src/plugins/intel_gpu/tests/functional/remote_tensor_tests/ocl_remote_tensor_tests.cpp +++ b/src/plugins/intel_gpu/tests/functional/remote_tensor_tests/ocl_remote_tensor_tests.cpp @@ -5,6 +5,8 @@ #ifdef OV_GPU_WITH_OCL_RT #include +#include +#include #include "openvino/core/preprocess/pre_post_process.hpp" #include "openvino/op/add.hpp" @@ -15,6 +17,7 @@ #include "openvino/runtime/intel_gpu/properties.hpp" #include "openvino/runtime/remote_tensor.hpp" #include "openvino/util/memory.hpp" +#include "openvino/util/mmap_object.hpp" #include "remote_tensor_tests/helpers.hpp" #include "common_test_utils/ov_tensor_utils.hpp" @@ -3104,4 +3107,161 @@ TEST(GpuRemoteTensorFromCpu, smoke_allocAlignedCPUMemory) { ov::util::aligned_free(output_ptr); } + +using MmapFileMemoryParams = std::tuple; + +class GpuRemoteTensorFromFile : public ::testing::TestWithParam { +public: + static std::string getTestCaseName(const testing::TestParamInfo& obj) { + const auto& [offset, bytes_after_offset] = obj.param; + return "offset_" + std::to_string(offset) + "_bytes_after_offset_" + std::to_string(bytes_after_offset); + } + +protected: + std::filesystem::path m_file_path; + + void SetUp() override { + m_file_path = ov::test::utils::generateTestFilePrefix() + ".bin"; + } + + void TearDown() override { + std::error_code ec; + std::filesystem::remove(m_file_path, ec); + } + + static void write_data_at_offset(const std::filesystem::path& path, + std::size_t offset, + const std::vector& values) { + std::ofstream file(path, std::ios::binary); + if (offset > 0) { + const std::vector padding(offset, 0); + file.write(padding.data(), padding.size()); + } + file.write(reinterpret_cast(values.data()), values.size() * sizeof(float)); + } + + static std::vector make_values(std::size_t element_count) { + std::vector values(element_count); + for (std::size_t i = 0; i < element_count; ++i) { + values[i] = static_cast(i + 1); + } + return values; + } +}; + +TEST_P(GpuRemoteTensorFromFile, smoke_mmapFileMemoryAsInput) { + const auto& [offset, bytes_after_offset] = GetParam(); + + ov::Core core; + std::string target_device = ov::test::utils::DEVICE_GPU; + const ov::Shape shape{bytes_after_offset / sizeof(float)}; + const size_t element_count = ov::shape_size(shape); + auto ctx = core.get_default_context(target_device).as(); + + const auto input_values = make_values(element_count); + write_data_at_offset(m_file_path, offset, input_values); + ASSERT_EQ(std::filesystem::file_size(m_file_path), offset + bytes_after_offset); + + const std::size_t cacheline = core.get_property(target_device, ov::intel_gpu::cacheline_size); + const std::size_t output_buffer_size = ((bytes_after_offset + cacheline - 1) / cacheline) * cacheline; + void* output_ptr = ov::util::aligned_alloc(output_buffer_size, cacheline); + std::fill_n(static_cast(output_ptr), output_buffer_size, 0); + + { + auto remote_input_tensor = ctx.create_tensor( + ov::element::f32, + shape, + ov::intel_gpu::FileDescriptor{m_file_path, offset, ov::intel_gpu::FileAccess::READ}); + ASSERT_TRUE(remote_input_tensor.is()); + auto remote_output_tensor = + ctx.create_tensor(ov::element::f32, + shape, + ov::intel_gpu::VirtualAddressMemory(output_ptr, static_cast(output_buffer_size))); + + auto model = make_copy_model(shape); + auto compiled = core.compile_model(model, ctx); + auto infer_req = compiled.create_infer_request(); + infer_req.set_tensor(compiled.input(), remote_input_tensor); + infer_req.set_tensor(compiled.output(), remote_output_tensor); + infer_req.infer(); + + for (size_t i = 0; i < element_count; ++i) { + EXPECT_FLOAT_EQ(static_cast(output_ptr)[i], input_values[i]) << "Mismatch at index " << i; + } + } + + ov::util::aligned_free(output_ptr); +} + +TEST_P(GpuRemoteTensorFromFile, smoke_mmapFileMemoryAsOutput) { + const auto& [offset, bytes_after_offset] = GetParam(); + + ov::Core core; + std::string target_device = ov::test::utils::DEVICE_GPU; + const ov::Shape shape{bytes_after_offset / sizeof(float)}; + const size_t element_count = ov::shape_size(shape); + auto ctx = core.get_default_context(target_device).as(); + + const auto input_values = make_values(element_count); + write_data_at_offset(m_file_path, offset, std::vector(element_count, 0.0f)); + ASSERT_EQ(std::filesystem::file_size(m_file_path), offset + bytes_after_offset); + + { + auto remote_output_tensor = ctx.create_tensor( + ov::element::f32, + shape, + ov::intel_gpu::FileDescriptor{m_file_path, offset, ov::intel_gpu::FileAccess::READ_WRITE}); + + auto model = make_copy_model(shape); + auto compiled = core.compile_model(model, ctx); + auto infer_req = compiled.create_infer_request(); + infer_req.set_tensor(compiled.input(), + ov::Tensor(ov::element::f32, shape, const_cast(input_values.data()))); + infer_req.set_tensor(compiled.output(), remote_output_tensor); + infer_req.infer(); + } + + std::vector file_content(element_count, 0.0f); + std::ifstream file(m_file_path, std::ios::binary); + file.seekg(offset); + file.read(reinterpret_cast(file_content.data()), bytes_after_offset); + for (size_t i = 0; i < element_count; ++i) { + EXPECT_FLOAT_EQ(file_content[i], input_values[i]) << "Mismatch in file at index " << i; + } +} + +static std::vector generate_mmap_file_memory_params() { +#ifdef _WIN32 + // Windows maps file views with 64K allocation granularity + const std::size_t mmap_granularity = 65536; +#else + // Page size varies per platform: 4K on x86-64, 16K or 64K on some ARM64 Linux distributions. + const auto mmap_granularity = static_cast(ov::util::get_system_page_size()); +#endif + const std::vector> layouts{ + // Sizes smaller than / not divisible by the device cacheline size, e.g. f32 with shape {1}. + {0, sizeof(float)}, + {mmap_granularity, 3 * sizeof(float)}, + {0, 256}, + {0, 4 * mmap_granularity}, + {mmap_granularity, 256}, + {2 * mmap_granularity, 256}, + {16 * mmap_granularity, 256}, + {mmap_granularity, mmap_granularity}, + {3 * mmap_granularity, 4 * mmap_granularity}, + {8 * mmap_granularity, 16 * mmap_granularity}}; + + std::vector params; + params.reserve(layouts.size()); + for (const auto& [offset, bytes_after_offset] : layouts) { + params.emplace_back(offset, bytes_after_offset); + } + return params; +} + +INSTANTIATE_TEST_SUITE_P(smoke_mmapFileMemory, + GpuRemoteTensorFromFile, + ::testing::ValuesIn(generate_mmap_file_memory_params()), + GpuRemoteTensorFromFile::getTestCaseName); + #endif // OV_GPU_WITH_OCL_RT