Reduce IR Constant Memory Retention #35457
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,128 @@ | ||||||
| // Copyright (C) 2018-2026 Intel Corporation | ||||||
| // SPDX-License-Identifier: Apache-2.0 | ||||||
| // | ||||||
|
|
||||||
| #pragma once | ||||||
|
|
||||||
| #include <filesystem> | ||||||
| #include <map> | ||||||
| #include <memory> | ||||||
|
|
||||||
| namespace ov { | ||||||
| class AlignedBuffer; | ||||||
| } // namespace ov | ||||||
|
|
||||||
| namespace ov::util { | ||||||
|
|
||||||
| /** | ||||||
| * @brief Interface for loading weight data regions from an underlying storage. | ||||||
| * | ||||||
| * Implementations may serve weights either from an in-memory buffer or from a | ||||||
| * file-backed source. | ||||||
| */ | ||||||
| class WeightsProvider { | ||||||
| public: | ||||||
| virtual ~WeightsProvider() = default; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Make a contiguous region of weights. | ||||||
| * | ||||||
| * @param offset Byte offset from the beginning of the weights source. | ||||||
| * @param size Number of bytes to load. | ||||||
| * @return Buffer containing the requested weights region. | ||||||
| */ | ||||||
| virtual std::shared_ptr<ov::AlignedBuffer> make_region(size_t offset, size_t size) = 0; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns the total size of the weights source in bytes. | ||||||
| * | ||||||
| * @return Size of the underlying weights source. | ||||||
| */ | ||||||
| virtual size_t size() const = 0; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns the path to the weights source when the provider is file-backed. | ||||||
| * | ||||||
| * @return Path to the weights source, or an empty path if the provider is not file-backed. | ||||||
| */ | ||||||
| virtual std::filesystem::path path() const; | ||||||
|
praasz marked this conversation as resolved.
|
||||||
| }; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Weights provider implementation backed by an already allocated buffer. | ||||||
| */ | ||||||
| class BufferWeightsProvider : public WeightsProvider { | ||||||
| public: | ||||||
| /** | ||||||
| * @brief Constructs a weights provider over an existing buffer. | ||||||
| * | ||||||
| * @param weights Buffer containing the full weights blob. | ||||||
| */ | ||||||
| explicit BufferWeightsProvider(std::shared_ptr<ov::AlignedBuffer> weights); | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns a view of the requested region from the backing buffer. | ||||||
| * | ||||||
| * @param offset Byte offset from the beginning of the weights buffer. | ||||||
| * @param size Number of bytes to expose. | ||||||
| * @return Buffer referencing the requested region. | ||||||
| */ | ||||||
| std::shared_ptr<ov::AlignedBuffer> make_region(size_t offset, size_t size) override; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns the total size of the backing weights buffer in bytes. | ||||||
| * | ||||||
| * @return Size of the underlying buffer. | ||||||
| */ | ||||||
| size_t size() const override; | ||||||
|
|
||||||
| private: | ||||||
| std::shared_ptr<ov::AlignedBuffer> m_weights; | ||||||
| }; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Weights provider implementation backed by a weights file on disk. | ||||||
| */ | ||||||
| class FileWeightsProvider : public WeightsProvider { | ||||||
| public: | ||||||
| /** | ||||||
| * @brief Constructs a weights provider for the specified file. | ||||||
| * | ||||||
| * @param weights_path Path to the weights file. | ||||||
| */ | ||||||
| explicit FileWeightsProvider(std::filesystem::path weights_path); | ||||||
|
|
||||||
| /** | ||||||
| * @brief Loads the requested region from the weights file. | ||||||
| * | ||||||
| * Implementations may cache previously loaded regions. | ||||||
| * | ||||||
| * @param offset Byte offset from the beginning of the weights file. | ||||||
| * @param size Number of bytes to load. | ||||||
| * @return Buffer containing the requested file region. | ||||||
| */ | ||||||
| std::shared_ptr<ov::AlignedBuffer> make_region(size_t offset, size_t size) override; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns the total size of the weights file in bytes. | ||||||
| * | ||||||
| * @return Size of the file-backed weights source. | ||||||
| */ | ||||||
| size_t size() const override; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns the path to the backing weights file. | ||||||
| * | ||||||
| * @return Path to the backing weights file. | ||||||
| */ | ||||||
| std::filesystem::path path() const override; | ||||||
|
|
||||||
| private: | ||||||
| using WeightsRegionKey = std::pair<size_t, size_t>; | ||||||
|
praasz marked this conversation as resolved.
|
||||||
|
|
||||||
| std::filesystem::path m_weights_path{}; | ||||||
| size_t m_weights_size{}; | ||||||
| // Cache of previously loaded weights regions, keyed by (offset, size) of the region in the weights file. | ||||||
| std::map<WeightsRegionKey, std::shared_ptr<ov::AlignedBuffer>> m_loaded_weights_regions; | ||||||
|
Contributor
There was a problem hiding this comment. For futher analysis why not use Constant ID (buffer) for it?
Contributor
There was a problem hiding this comment. [MEDIUM] you only add new buffers into m_loaded_weights_regions but never clean unused buffers. I'd suggest to store them as weak_ptr and clean expired ones regularly.
Suggested change
|
||||||
| }; | ||||||
| } // namespace ov::util | ||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| // Copyright (C) 2018-2026 Intel Corporation | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| #include "openvino/xml_util/weights_provider.hpp" | ||
|
|
||
| #include <fstream> | ||
|
|
||
| #include "openvino/runtime/aligned_buffer.hpp" | ||
| #include "openvino/runtime/shared_buffer.hpp" | ||
| #include "openvino/util/common_util.hpp" | ||
| #include "openvino/util/file_util.hpp" | ||
| #include "openvino/util/mmap_object.hpp" | ||
|
|
||
| namespace ov::util { | ||
|
|
||
| namespace { | ||
|
|
||
| size_t get_mmap_region_threshold() { | ||
| const auto page_size = ov::util::get_system_page_size(); | ||
| return page_size > 0 ? static_cast<size_t>(page_size) : 4096; | ||
|
Contributor
There was a problem hiding this comment. Why use some value if page size is 0? Will it be even possible? |
||
| } | ||
|
|
||
| } // namespace | ||
|
|
||
| std::filesystem::path WeightsProvider::path() const { | ||
| return {}; | ||
| } | ||
|
|
||
| BufferWeightsProvider::BufferWeightsProvider(std::shared_ptr<ov::AlignedBuffer> weights) | ||
| : m_weights(std::move(weights)) {} | ||
|
|
||
| std::shared_ptr<ov::AlignedBuffer> BufferWeightsProvider::make_region(size_t offset, size_t size) { | ||
| OPENVINO_ASSERT(m_weights != nullptr, "Empty weights data in bin file or bin file cannot be found!"); | ||
|
t-jankowski marked this conversation as resolved.
|
||
| OPENVINO_ASSERT(offset <= m_weights->size() && size <= m_weights->size() - offset, | ||
| "Incorrect weights in bin file!"); | ||
|
|
||
| auto* data = m_weights->get_ptr<char>() + offset; | ||
| return std::make_shared<ov::SharedBuffer<std::shared_ptr<ov::AlignedBuffer>>>(data, size, m_weights); | ||
| } | ||
|
|
||
| size_t BufferWeightsProvider::size() const { | ||
| return m_weights->size(); | ||
| } | ||
|
|
||
| FileWeightsProvider::FileWeightsProvider(std::filesystem::path weights_path) | ||
| : m_weights_path(std::move(weights_path)), | ||
| m_weights_size(ov::util::file_size(m_weights_path)) { | ||
| std::ifstream weights_stream(m_weights_path, std::ios::binary); | ||
| OPENVINO_ASSERT(weights_stream.is_open(), m_weights_path, " cannot be opened"); | ||
| } | ||
|
|
||
| std::shared_ptr<ov::AlignedBuffer> FileWeightsProvider::make_region(size_t offset, size_t size) { | ||
| OPENVINO_ASSERT(offset <= m_weights_size && size <= m_weights_size - offset, "Incorrect weights in bin file!"); | ||
|
Contributor
There was a problem hiding this comment. Looks like common validation pattern, should some helper be created to return bool if offset, size are valid? |
||
|
|
||
| const FileWeightsProvider::WeightsRegionKey key{offset, size}; | ||
| if (const auto found = m_loaded_weights_regions.find(key); found != m_loaded_weights_regions.end()) { | ||
| return found->second; | ||
| } | ||
|
|
||
| std::shared_ptr<ov::AlignedBuffer> buffer; | ||
| if (size >= get_mmap_region_threshold()) { | ||
| auto mapped_memory = ov::load_mmap_object(m_weights_path, offset, size); | ||
| buffer = std::make_shared<ov::SharedBuffer<std::shared_ptr<ov::MappedMemory>>>(mapped_memory->data(), | ||
| mapped_memory->size(), | ||
| mapped_memory); | ||
| } else { | ||
| auto file_region = std::make_shared<ov::AlignedBuffer>(size); | ||
| if (size > 0) { | ||
| std::ifstream weights_stream(m_weights_path, std::ios::binary); | ||
| OPENVINO_ASSERT(weights_stream.is_open(), m_weights_path, " cannot be opened"); | ||
| weights_stream.seekg(static_cast<std::streamoff>(offset), std::ios::beg); | ||
| weights_stream.read(file_region->get_ptr<char>(), static_cast<std::streamsize>(size)); | ||
| OPENVINO_ASSERT(weights_stream, "Failed to read weights from ", m_weights_path); | ||
| } | ||
| buffer = std::make_shared<ov::SharedBuffer<std::shared_ptr<ov::AlignedBuffer>>>(file_region->get_ptr<char>(), | ||
| size, | ||
| file_region); | ||
| } | ||
|
|
||
| m_loaded_weights_regions.emplace(key, buffer); | ||
|
|
||
| return buffer; | ||
| } | ||
|
|
||
| size_t FileWeightsProvider::size() const { | ||
| return m_weights_size; | ||
| } | ||
|
|
||
| std::filesystem::path FileWeightsProvider::path() const { | ||
| return m_weights_path; | ||
| } | ||
| } // namespace ov::util | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.