Reduce IR Constant Memory Retention #35457
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,52 @@ | ||||||
| // Copyright (C) 2018-2026 Intel Corporation | ||||||
| // SPDX-License-Identifier: Apache-2.0 | ||||||
| // | ||||||
|
|
||||||
| #pragma once | ||||||
|
|
||||||
| #include <filesystem> | ||||||
| #include <map> | ||||||
| #include <memory> | ||||||
|
|
||||||
| namespace ov { | ||||||
| class AlignedBuffer; | ||||||
| } // namespace ov | ||||||
|
|
||||||
| namespace ov::util { | ||||||
|
|
||||||
| class WeightsProvider { | ||||||
| public: | ||||||
| virtual ~WeightsProvider() = default; | ||||||
|
|
||||||
| virtual std::shared_ptr<ov::AlignedBuffer> load_region(size_t offset, size_t size) = 0; | ||||||
|
mlukasze marked this conversation as resolved.
Outdated
|
||||||
| virtual size_t size() const = 0; | ||||||
| }; | ||||||
|
|
||||||
| class BufferWeightsProvider : public WeightsProvider { | ||||||
| public: | ||||||
| explicit BufferWeightsProvider(std::shared_ptr<ov::AlignedBuffer> weights); | ||||||
|
|
||||||
| std::shared_ptr<ov::AlignedBuffer> load_region(size_t offset, size_t size) override; | ||||||
| size_t size() const override; | ||||||
|
|
||||||
| private: | ||||||
| std::shared_ptr<ov::AlignedBuffer> m_weights; | ||||||
| }; | ||||||
|
|
||||||
| class FileWeightsProvider : public WeightsProvider { | ||||||
| public: | ||||||
| explicit FileWeightsProvider(std::filesystem::path weights_path); | ||||||
|
|
||||||
| std::shared_ptr<ov::AlignedBuffer> load_region(size_t offset, size_t size) override; | ||||||
| size_t size() const override; | ||||||
|
|
||||||
| private: | ||||||
| using WeightsRegionKey = std::pair<size_t, size_t>; | ||||||
|
praasz marked this conversation as resolved.
|
||||||
|
|
||||||
| std::filesystem::path m_weights_path; | ||||||
| size_t m_weights_size = 0; | ||||||
|
barnasm1 marked this conversation as resolved.
Outdated
|
||||||
| size_t m_weights_source_id = 0; | ||||||
| std::shared_ptr<ov::AlignedBuffer> m_weights_source_handle; | ||||||
| std::map<WeightsRegionKey, std::shared_ptr<ov::AlignedBuffer>> m_loaded_weights_regions; | ||||||
|
Contributor
There was a problem hiding this comment. For futher analysis why not use Constant ID (buffer) for it?
Contributor
There was a problem hiding this comment. [MEDIUM] you only add new buffers into m_loaded_weights_regions but never clean unused buffers. I'd suggest to store them as weak_ptr and clean expired ones regularly.
Suggested change
|
||||||
| }; | ||||||
| } // namespace ov::util | ||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| // Copyright (C) 2018-2026 Intel Corporation | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| #include "openvino/xml_util/weights_provider.hpp" | ||
|
|
||
| #include <fstream> | ||
|
|
||
| #include "openvino/runtime/aligned_buffer.hpp" | ||
| #include "openvino/runtime/shared_buffer.hpp" | ||
| #include "openvino/util/common_util.hpp" | ||
| #include "openvino/util/file_util.hpp" | ||
|
|
||
| namespace ov::util { | ||
|
|
||
| namespace { | ||
|
|
||
| class FileRegionBuffer : public ov::AlignedBuffer { | ||
|
praasz marked this conversation as resolved.
Outdated
|
||
| public: | ||
| FileRegionBuffer(size_t size, size_t source_id, size_t offset, std::shared_ptr<ov::AlignedBuffer> source_handle) | ||
| : ov::AlignedBuffer(size), | ||
| m_source_handle(std::move(source_handle)), | ||
| m_descriptor(ov::create_base_descriptor(source_id, offset, m_source_handle)) {} | ||
|
|
||
| std::shared_ptr<ov::IBufferDescriptor> get_descriptor() const override { | ||
| return m_descriptor; | ||
| } | ||
|
|
||
| private: | ||
| std::shared_ptr<ov::AlignedBuffer> m_source_handle; | ||
| std::shared_ptr<ov::IBufferDescriptor> m_descriptor; | ||
| }; | ||
|
|
||
| } // namespace | ||
|
|
||
| BufferWeightsProvider::BufferWeightsProvider(std::shared_ptr<ov::AlignedBuffer> weights) | ||
| : m_weights(std::move(weights)) {} | ||
|
|
||
| std::shared_ptr<ov::AlignedBuffer> BufferWeightsProvider::load_region(size_t offset, size_t size) { | ||
| OPENVINO_ASSERT(m_weights != nullptr, "Empty weights data in bin file or bin file cannot be found!"); | ||
|
t-jankowski marked this conversation as resolved.
|
||
| OPENVINO_ASSERT(offset <= m_weights->size() && size <= m_weights->size() - offset, | ||
| "Incorrect weights in bin file!"); | ||
|
|
||
| auto* data = m_weights->get_ptr<char>() + offset; | ||
| return std::make_shared<ov::SharedBuffer<std::shared_ptr<ov::AlignedBuffer>>>(data, size, m_weights); | ||
| } | ||
|
|
||
| size_t BufferWeightsProvider::size() const { | ||
| return m_weights->size(); | ||
| } | ||
|
|
||
| FileWeightsProvider::FileWeightsProvider(std::filesystem::path weights_path) | ||
| : m_weights_path(std::move(weights_path)), | ||
| m_weights_size(ov::util::file_size(m_weights_path)), | ||
| m_weights_source_id(std::hash<std::decay_t<decltype(m_weights_path.native())>>{}(m_weights_path.native())), | ||
|
barnasm1 marked this conversation as resolved.
Outdated
|
||
| m_weights_source_handle(std::make_shared<ov::AlignedBuffer>()) { | ||
| std::ifstream weights_stream(m_weights_path, std::ios::binary); | ||
| OPENVINO_ASSERT(weights_stream.is_open(), "Weights file ", m_weights_path, " cannot be opened!"); | ||
|
mlukasze marked this conversation as resolved.
Outdated
|
||
| } | ||
|
|
||
| std::shared_ptr<ov::AlignedBuffer> FileWeightsProvider::load_region(size_t offset, size_t size) { | ||
| OPENVINO_ASSERT(offset <= m_weights_size && size <= m_weights_size - offset, "Incorrect weights in bin file!"); | ||
|
Contributor
There was a problem hiding this comment. Looks like common validation pattern, should some helper be created to return bool if offset, size are valid? |
||
|
|
||
| const FileWeightsProvider::WeightsRegionKey key{offset, size}; | ||
| if (const auto found = m_loaded_weights_regions.find(key); found != m_loaded_weights_regions.end()) { | ||
| return found->second; | ||
| } | ||
|
|
||
| std::ifstream weights_stream(m_weights_path, std::ios::binary); | ||
| OPENVINO_ASSERT(weights_stream.is_open(), "Weights file ", m_weights_path, " cannot be opened!"); | ||
|
|
||
| weights_stream.seekg(static_cast<std::streamoff>(offset), std::ios::beg); | ||
| OPENVINO_ASSERT(weights_stream.good(), "Failed to seek weights file ", m_weights_path, " to offset ", offset); | ||
|
|
||
| auto buffer = std::make_shared<FileRegionBuffer>(size, m_weights_source_id, offset, m_weights_source_handle); | ||
|
|
||
| weights_stream.read(buffer->get_ptr<char>(), static_cast<std::streamsize>(size)); | ||
| OPENVINO_ASSERT(static_cast<size_t>(weights_stream.gcount()) == size, | ||
| "Failed to read ", | ||
| size, | ||
| " bytes from weights file ", | ||
| m_weights_path, | ||
| " at offset ", | ||
| offset); | ||
|
|
||
| m_loaded_weights_regions.emplace(key, buffer); | ||
|
|
||
| return buffer; | ||
| } | ||
|
|
||
| size_t FileWeightsProvider::size() const { | ||
| return m_weights_size; | ||
| } | ||
| } // namespace ov::util | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.