Reduce IR Constant Memory Retention #35457
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,116 @@ | ||||||
| // Copyright (C) 2018-2026 Intel Corporation | ||||||
| // SPDX-License-Identifier: Apache-2.0 | ||||||
| // | ||||||
|
|
||||||
| #pragma once | ||||||
|
|
||||||
| #include <filesystem> | ||||||
| #include <map> | ||||||
| #include <memory> | ||||||
|
|
||||||
| namespace ov { | ||||||
| class AlignedBuffer; | ||||||
| } // namespace ov | ||||||
|
|
||||||
| namespace ov::util { | ||||||
|
|
||||||
| /** | ||||||
| * @brief Interface for loading weight data regions from an underlying storage. | ||||||
| * | ||||||
| * Implementations may serve weights either from an in-memory buffer or from a | ||||||
| * file-backed source. | ||||||
| */ | ||||||
| class WeightsProvider { | ||||||
| public: | ||||||
| virtual ~WeightsProvider() = default; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Make a contiguous region of weights. | ||||||
| * | ||||||
| * @param offset Byte offset from the beginning of the weights source. | ||||||
| * @param size Number of bytes to load. | ||||||
| * @return Buffer containing the requested weights region. | ||||||
| */ | ||||||
| virtual std::shared_ptr<ov::AlignedBuffer> make_region(size_t offset, size_t size) = 0; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns the total size of the weights source in bytes. | ||||||
| * | ||||||
| * @return Size of the underlying weights source. | ||||||
| */ | ||||||
| virtual size_t size() const = 0; | ||||||
| }; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Weights provider implementation backed by an already allocated buffer. | ||||||
| */ | ||||||
| class BufferWeightsProvider : public WeightsProvider { | ||||||
| public: | ||||||
| /** | ||||||
| * @brief Constructs a weights provider over an existing buffer. | ||||||
| * | ||||||
| * @param weights Buffer containing the full weights blob. | ||||||
| */ | ||||||
| explicit BufferWeightsProvider(std::shared_ptr<ov::AlignedBuffer> weights); | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns a view of the requested region from the backing buffer. | ||||||
| * | ||||||
| * @param offset Byte offset from the beginning of the weights buffer. | ||||||
| * @param size Number of bytes to expose. | ||||||
| * @return Buffer referencing the requested region. | ||||||
| */ | ||||||
| std::shared_ptr<ov::AlignedBuffer> make_region(size_t offset, size_t size) override; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns the total size of the backing weights buffer in bytes. | ||||||
| * | ||||||
| * @return Size of the underlying buffer. | ||||||
| */ | ||||||
| size_t size() const override; | ||||||
|
|
||||||
| private: | ||||||
| std::shared_ptr<ov::AlignedBuffer> m_weights; | ||||||
| }; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Weights provider implementation backed by a weights file on disk. | ||||||
| */ | ||||||
| class FileWeightsProvider : public WeightsProvider { | ||||||
| public: | ||||||
| /** | ||||||
| * @brief Constructs a weights provider for the specified file. | ||||||
| * | ||||||
| * @param weights_path Path to the weights file. | ||||||
| */ | ||||||
| explicit FileWeightsProvider(std::filesystem::path weights_path); | ||||||
|
|
||||||
| /** | ||||||
| * @brief Loads the requested region from the weights file. | ||||||
| * | ||||||
| * Implementations may cache previously loaded regions. | ||||||
| * | ||||||
| * @param offset Byte offset from the beginning of the weights file. | ||||||
| * @param size Number of bytes to load. | ||||||
| * @return Buffer containing the requested file region. | ||||||
| */ | ||||||
| std::shared_ptr<ov::AlignedBuffer> make_region(size_t offset, size_t size) override; | ||||||
|
|
||||||
| /** | ||||||
| * @brief Returns the total size of the weights file in bytes. | ||||||
| * | ||||||
| * @return Size of the file-backed weights source. | ||||||
| */ | ||||||
| size_t size() const override; | ||||||
|
|
||||||
| private: | ||||||
| using WeightsRegionKey = std::pair<size_t, size_t>; | ||||||
|
praasz marked this conversation as resolved.
|
||||||
|
|
||||||
| std::filesystem::path m_weights_path; | ||||||
| size_t m_weights_size = 0; | ||||||
|
barnasm1 marked this conversation as resolved.
Outdated
|
||||||
| size_t m_weights_source_id = 0; | ||||||
| std::shared_ptr<ov::AlignedBuffer> m_weights_source_handle; | ||||||
| // Cache of previously loaded weights regions, keyed by (offset, size) of the region in the weights file. | ||||||
| std::map<WeightsRegionKey, std::shared_ptr<ov::AlignedBuffer>> m_loaded_weights_regions; | ||||||
|
Contributor
There was a problem hiding this comment. For futher analysis why not use Constant ID (buffer) for it?
Contributor
There was a problem hiding this comment. [MEDIUM] you only add new buffers into m_loaded_weights_regions but never clean unused buffers. I'd suggest to store them as weak_ptr and clean expired ones regularly.
Suggested change
|
||||||
| }; | ||||||
| } // namespace ov::util | ||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| // Copyright (C) 2018-2026 Intel Corporation | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| #include "openvino/xml_util/weights_provider.hpp" | ||
|
|
||
| #include <fstream> | ||
|
|
||
| #include "openvino/runtime/aligned_buffer.hpp" | ||
| #include "openvino/runtime/lazy_buffer.hpp" | ||
| #include "openvino/runtime/shared_buffer.hpp" | ||
| #include "openvino/util/common_util.hpp" | ||
| #include "openvino/util/file_util.hpp" | ||
|
|
||
| namespace ov::util { | ||
|
|
||
| namespace { | ||
|
|
||
| class FileRegionBuffer : public ov::LazyBuffer { | ||
| public: | ||
| FileRegionBuffer(std::filesystem::path file_path, | ||
| size_t size, | ||
| size_t source_id, | ||
| size_t offset, | ||
| std::shared_ptr<ov::AlignedBuffer> source_handle) | ||
| : ov::LazyBuffer(std::move(file_path), offset, size), | ||
| m_source_handle(std::move(source_handle)), | ||
| m_descriptor(ov::create_base_descriptor(source_id, offset, m_source_handle)) {} | ||
|
|
||
| std::shared_ptr<ov::IBufferDescriptor> get_descriptor() const override { | ||
| return m_descriptor; | ||
| } | ||
|
|
||
| private: | ||
| std::shared_ptr<ov::AlignedBuffer> m_source_handle; | ||
| std::shared_ptr<ov::IBufferDescriptor> m_descriptor; | ||
| }; | ||
|
|
||
| } // namespace | ||
|
|
||
| BufferWeightsProvider::BufferWeightsProvider(std::shared_ptr<ov::AlignedBuffer> weights) | ||
| : m_weights(std::move(weights)) {} | ||
|
|
||
| std::shared_ptr<ov::AlignedBuffer> BufferWeightsProvider::make_region(size_t offset, size_t size) { | ||
| OPENVINO_ASSERT(m_weights != nullptr, "Empty weights data in bin file or bin file cannot be found!"); | ||
|
t-jankowski marked this conversation as resolved.
|
||
| OPENVINO_ASSERT(offset <= m_weights->size() && size <= m_weights->size() - offset, | ||
| "Incorrect weights in bin file!"); | ||
|
|
||
| auto* data = m_weights->get_ptr<char>() + offset; | ||
| return std::make_shared<ov::SharedBuffer<std::shared_ptr<ov::AlignedBuffer>>>(data, size, m_weights); | ||
| } | ||
|
|
||
| size_t BufferWeightsProvider::size() const { | ||
| return m_weights->size(); | ||
| } | ||
|
|
||
| FileWeightsProvider::FileWeightsProvider(std::filesystem::path weights_path) | ||
| : m_weights_path(std::move(weights_path)), | ||
| m_weights_size(ov::util::file_size(m_weights_path)), | ||
| m_weights_source_id(std::filesystem::hash_value(weights_path)), | ||
| m_weights_source_handle(std::make_shared<ov::AlignedBuffer>()) { | ||
| std::ifstream weights_stream(m_weights_path, std::ios::binary); | ||
| OPENVINO_ASSERT(weights_stream.is_open(), m_weights_path, " cannot be opened"); | ||
| } | ||
|
|
||
| std::shared_ptr<ov::AlignedBuffer> FileWeightsProvider::make_region(size_t offset, size_t size) { | ||
| OPENVINO_ASSERT(offset <= m_weights_size && size <= m_weights_size - offset, "Incorrect weights in bin file!"); | ||
|
Contributor
There was a problem hiding this comment. Looks like common validation pattern, should some helper be created to return bool if offset, size are valid? |
||
|
|
||
| const FileWeightsProvider::WeightsRegionKey key{offset, size}; | ||
| if (const auto found = m_loaded_weights_regions.find(key); found != m_loaded_weights_regions.end()) { | ||
| return found->second; | ||
| } | ||
|
|
||
| auto buffer = | ||
| std::make_shared<FileRegionBuffer>(m_weights_path, size, m_weights_source_id, offset, m_weights_source_handle); | ||
|
|
||
| m_loaded_weights_regions.emplace(key, buffer); | ||
|
|
||
| return buffer; | ||
| } | ||
|
|
||
| size_t FileWeightsProvider::size() const { | ||
| return m_weights_size; | ||
| } | ||
| } // namespace ov::util | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.