From f1234f3856d9196c7a64729e464503bd33f8ed95 Mon Sep 17 00:00:00 2001 From: sunxiaoxia2022 Date: Tue, 16 Jun 2026 16:26:40 +0800 Subject: [PATCH 1/7] fix CVS-187773, add withDnnlDescriptorConcurrency for setting real workers to calculate scratchpad memory size --- src/plugins/intel_cpu/src/cpu_parallel.hpp | 55 ++++++- src/plugins/intel_cpu/src/graph.cpp | 6 +- src/plugins/intel_cpu/src/node.cpp | 2 +- src/plugins/intel_cpu/src/nodes/deconv.cpp | 40 ++--- .../dnnl/dnnl_convolution_primitive.cpp | 153 +++++++----------- .../dnnl/dnnl_fullyconnected_primitive.cpp | 39 ++--- .../executors/dnnl/dnnl_matmul_primitive.cpp | 101 ++++++------ src/plugins/intel_cpu/src/nodes/lrn.cpp | 37 +++-- src/plugins/intel_cpu/src/nodes/pooling.cpp | 43 ++--- src/plugins/intel_cpu/src/nodes/rnn.cpp | 23 +-- src/plugins/intel_cpu/src/nodes/softmax.cpp | 59 +++---- 11 files changed, 302 insertions(+), 256 deletions(-) diff --git a/src/plugins/intel_cpu/src/cpu_parallel.hpp b/src/plugins/intel_cpu/src/cpu_parallel.hpp index f64f53911d68f1..415b5fd7d8daef 100644 --- a/src/plugins/intel_cpu/src/cpu_parallel.hpp +++ b/src/plugins/intel_cpu/src/cpu_parallel.hpp @@ -12,6 +12,54 @@ namespace ov::intel_cpu { class ThreadPool; +#if OV_THREAD == OV_THREAD_TBB_ADAPTIVE +class DnnlMaxConcurrencyGuard { +public: + explicit DnnlMaxConcurrencyGuard(int max_concurrency) { + if (dnnl_threadpool_interop_get_max_concurrency(&m_prev_max_concurrency) != dnnl_success) { + return; + } + + if (m_prev_max_concurrency == max_concurrency) { + return; + } + + if (dnnl_threadpool_interop_set_max_concurrency(max_concurrency) == dnnl_success) { + m_restore_required = true; + } + } + + ~DnnlMaxConcurrencyGuard() { + if (m_restore_required) { + dnnl_threadpool_interop_set_max_concurrency(m_prev_max_concurrency); + } + } + +private: + int m_prev_max_concurrency = 0; + bool m_restore_required = false; +}; + +inline int getDnnlMaxConcurrency() { + int max_concurrency = 0; + if (dnnl_threadpool_interop_get_max_concurrency(&max_concurrency) != dnnl_success) { + return 0; + } + return max_concurrency; +} + +template +decltype(auto) withDnnlDescriptorConcurrency(F&& f) { + DnnlMaxConcurrencyGuard guard(CpuParallel::get_num_worker_threads()); + return std::forward(f)(); +} +#else +template +decltype(auto) withDnnlDescriptorConcurrency(F&& f) { + return std::forward(f)(); +} +#endif + class CpuParallel { public: // Default multiplier for the number of virtual threads when tbb partitioner is AUTO. This value is determined @@ -41,7 +89,12 @@ class CpuParallel { [[nodiscard]] static int get_num_worker_threads() { return parallel_get_max_threads(); } - void activate() const { + void activateForInit() const { +#if OV_THREAD == OV_THREAD_TBB_ADAPTIVE + dnnl_threadpool_interop_set_max_concurrency(get_num_worker_threads()); +#endif + } + void activateForExecution() const { #if OV_THREAD == OV_THREAD_TBB_ADAPTIVE dnnl_threadpool_interop_set_max_concurrency(get_num_threads()); #endif diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index 735fd3d204d480..b1bfe03768fa82 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -125,7 +125,7 @@ void Graph::Init(const std::vector& graphNodes, m_context = context; m_stream = make_stream(getEngine(), m_context->getCpuParallel()->get_thread_pool()); - m_context->getCpuParallel()->activate(); + m_context->getCpuParallel()->activateForInit(); this->_name = std::move(name); @@ -382,7 +382,7 @@ void Graph::Init(const std::shared_ptr& model, m_context = context; m_stream = make_stream(getEngine(), m_context->getCpuParallel()->get_thread_pool()); - m_context->getCpuParallel()->activate(); + m_context->getCpuParallel()->activateForInit(); Replicate(model, inputConfigs, outputConfigs); @@ -394,6 +394,8 @@ void Graph::Activate() { // the allocation context collection from the outer graph so the state for inner graph is "Ready" // We probably want to avoid such uncertainty // OPENVINO_ASSERT(status == Status::Initialized, "Invalid graph status: ", static_cast(status)); + m_context->getCpuParallel()->activateForExecution(); + Allocate(); CreatePrimitivesAndExecConstants(); diff --git a/src/plugins/intel_cpu/src/node.cpp b/src/plugins/intel_cpu/src/node.cpp index e4dff3e0f8d4fc..c806c60bdeb0f3 100644 --- a/src/plugins/intel_cpu/src/node.cpp +++ b/src/plugins/intel_cpu/src/node.cpp @@ -808,7 +808,7 @@ void Node::updateDynamicParams() { getName(), " ", getOriginalLayers()); - context->getCpuParallel()->activate(); + context->getCpuParallel()->activateForExecution(); prepareParams(); } } diff --git a/src/plugins/intel_cpu/src/nodes/deconv.cpp b/src/plugins/intel_cpu/src/nodes/deconv.cpp index 07eea910122f06..d557bd97cc8ed5 100644 --- a/src/plugins/intel_cpu/src/nodes/deconv.cpp +++ b/src/plugins/intel_cpu/src/nodes/deconv.cpp @@ -24,6 +24,7 @@ #include "common/primitive_hashing_utils.hpp" #include "cpu/x64/cpu_isa_traits.hpp" #include "cpu_memory.h" +#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "dnnl_postops_composer_legacy.h" @@ -829,35 +830,37 @@ dnnl::primitive_desc createDescriptorInternal(const dnnl::memory::desc& in_candi const ov::CoordinateDiff& paddingR, const dnnl::primitive_attr& attr, const dnnl::engine& engine) { - auto convertDims = [](const std::vector& orig_dims) { - return memory::dims(orig_dims.begin(), orig_dims.end()); - }; + return withDnnlDescriptorConcurrency([&]() -> dnnl::primitive_desc { + auto convertDims = [](const std::vector& orig_dims) { + return memory::dims(orig_dims.begin(), orig_dims.end()); + }; - if (with_bias) { + if (with_bias) { + return dnnl::deconvolution_forward::primitive_desc(engine, + prop_kind::forward_inference, + dnnl::algorithm::deconvolution_direct, + in_candidate, + wgh_candidate, + bias_candidate, + out_candidate, + convertDims(stride), + convertDims(dilation), + convertDims(paddingL), + convertDims(paddingR), + attr); + } return dnnl::deconvolution_forward::primitive_desc(engine, prop_kind::forward_inference, dnnl::algorithm::deconvolution_direct, in_candidate, wgh_candidate, - bias_candidate, out_candidate, convertDims(stride), convertDims(dilation), convertDims(paddingL), convertDims(paddingR), attr); - } - return dnnl::deconvolution_forward::primitive_desc(engine, - prop_kind::forward_inference, - dnnl::algorithm::deconvolution_direct, - in_candidate, - wgh_candidate, - out_candidate, - convertDims(stride), - convertDims(dilation), - convertDims(paddingL), - convertDims(paddingR), - attr); + }); } } // namespace @@ -1044,8 +1047,9 @@ void Deconvolution::prepareParams() { selected_pd->getImplementationType()}; auto engine = getEngine(); + const auto worker_threads = context->getCpuParallel()->get_num_worker_threads(); - auto builder = [&engine](const DeconvKey& key) -> executorPtr { + auto builder = [&engine, worker_threads](const DeconvKey& key) -> executorPtr { dnnl::primitive_desc desc; convolution_forward::primitive_desc fwd_conv_pd; dnnl::memory::desc dnnlBiasDesc; diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp index a990b2c7f23b7e..20dda6f0e95568 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp @@ -24,6 +24,7 @@ #include #include +#include "cpu_parallel.hpp" #include "cpu/x64/cpu_isa_traits.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" @@ -338,109 +339,77 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, bool fcSemantic, const std::vector& implPriorities, const impl_desc_type defaultImplType) { - auto createPrimitiveDescriptor = [&](const dnnl::primitive_attr& attr) { - return fcSemantic ? createInnerProductDescriptor(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine) - : createConvolutionDescriptor(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - }; + return withDnnlDescriptorConcurrency([&]() -> primitive_desc { + auto createAnyDescriptor = [&]() -> dnnl::convolution_forward::primitive_desc { + auto inputDescAny = dnnl::memory::desc(inputDesc.get_dims(), inputDesc.get_data_type(), memory::format_tag::any); + auto outputDescAny = + dnnl::memory::desc(outputDesc.get_dims(), outputDesc.get_data_type(), memory::format_tag::any); + return createConvolutionDescriptor(inputDescAny, + weightDesc, + biasDesc, + outputDescAny, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); + }; + + auto createSelectedDescriptor = [&]() -> dnnl::convolution_forward::primitive_desc { + if (fcSemantic) { + return createInnerProductDescriptor(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); + } - auto createConvolutionDescriptorAny = [](const dnnl::memory::desc& inputDesc, - const dnnl::memory::desc& weightDesc, - const dnnl::memory::desc& biasDesc, - const dnnl::memory::desc& outputDesc, - const std::vector& stride, - const std::vector& dilation, - const std::vector& paddingL, - const std::vector& paddingR, - const dnnl::primitive_attr& attr, - const dnnl::engine& engine) { - auto inputDescAny = - dnnl::memory::desc(inputDesc.get_dims(), inputDesc.get_data_type(), memory::format_tag::any); - auto outputDescAny = - dnnl::memory::desc(outputDesc.get_dims(), outputDesc.get_data_type(), memory::format_tag::any); - return createConvolutionDescriptor(inputDescAny, - weightDesc, - biasDesc, - outputDescAny, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - }; + return createConvolutionDescriptor(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); + }; + + auto prim_desc = createSelectedDescriptor(); + auto first_desc = prim_desc; + if (defaultImplType == impl_desc_type::undef) { + if (!prim_desc) { + return createAnyDescriptor(); + } - auto prim_desc = createPrimitiveDescriptor(attr); - // keep first implementation descriptor to fallback to it if no other implementation is found - auto first_desc = prim_desc; - // if default implementation type is not specified, try to find the best implementation - if (defaultImplType == impl_desc_type::undef) { - if (!prim_desc) { - // fallback to 'any' implementation - return createConvolutionDescriptorAny(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - } + for (auto preferredImplType : implPriorities) { + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, preferredImplType); - for (auto preferredImplType : implPriorities) { - // the only way to fully reset primitive_desc after iterating over the implementations is to re-create it - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, preferredImplType); + if (found) { + return std::move(prim_desc); + } - if (found) { - return std::move(prim_desc); + prim_desc = createSelectedDescriptor(); } - prim_desc = createPrimitiveDescriptor(attr); + return std::move(first_desc); } - return std::move(first_desc); - } - // try to use a default implementations type (created using dummy shapes) if specified - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); - if (found) { - return std::move(prim_desc); - } + if (found) { + return std::move(prim_desc); + } - if (fcSemantic) { // fallback to the first implementation if used as FC executor return std::move(first_desc); - } - - // fallback to 'any' implementation - return createConvolutionDescriptorAny(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); + }); } static std::vector createPrimitiveAttrs(const ConvAttrs& attrs, diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp index e358d24e5e0626..07486ddd6391e6 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp @@ -19,6 +19,7 @@ #include #include "config.h" +#include "cpu_parallel.hpp" #include "cpu/x64/cpu_isa_traits.hpp" #include "cpu_memory.h" #include "cpu_types.h" @@ -354,26 +355,28 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, const std::vector& implPriorities, const bool useSparseWeights, const bool useWeightsDecompression) { - auto prim_desc = createDescriptorInternal(inputDesc, - weightDesc, - biasDesc, - outputDesc, - attr, - engine, - useSparseWeights, - useWeightsDecompression); - OPENVINO_ASSERT(prim_desc, "Failed to create inner_product primitive descriptor"); - auto first_desc = dnnl::inner_product_forward::primitive_desc(prim_desc.get()); + return withDnnlDescriptorConcurrency([&]() -> primitive_desc { + auto prim_desc = createDescriptorInternal(inputDesc, + weightDesc, + biasDesc, + outputDesc, + attr, + engine, + useSparseWeights, + useWeightsDecompression); + OPENVINO_ASSERT(prim_desc, "Failed to create inner_product primitive descriptor"); + auto first_desc = dnnl::inner_product_forward::primitive_desc(prim_desc.get()); + + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, [&](impl_desc_type implType) { + return contains(implPriorities, implType); + }); + + if (found) { + return std::move(prim_desc); + } - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, [&](impl_desc_type implType) { - return contains(implPriorities, implType); + return std::move(first_desc); }); - - if (found) { - return std::move(prim_desc); - } - - return std::move(first_desc); } static VectorDims makeDummyInputDims(const Shape& inShape, const Shape& wShape) { diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp index f45604af0971c8..cdb1c5cb93a9cf 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp @@ -20,6 +20,7 @@ #include #include "cpu_memory.h" +#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "dnnl_postops_composer.h" @@ -337,66 +338,68 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, [[maybe_unused]] const bool useSparseWeights, const bool useWeightsDecompression, const bool fcSemantic) { - auto createDescriptor = [&]() { - return fcSemantic ? createDescriptorInternalAsFc(inputDesc, + return withDnnlDescriptorConcurrency([&]() -> primitive_desc { + auto createDescriptor = [&]() { + return fcSemantic ? createDescriptorInternalAsFc(inputDesc, + weightDesc, + biasDesc, + outputDesc, + attr, + engine, + useWeightsDecompression) + : createDescriptorInternal(inputDesc, weightDesc, biasDesc, outputDesc, attr, engine, - useWeightsDecompression) - : createDescriptorInternal(inputDesc, - weightDesc, - biasDesc, - outputDesc, - attr, - engine, - transposeA, - transposeB); - }; - if (defaultImplType == impl_desc_type::undef) { - struct PrimitiveDescWithPriority { - dnnl::primitive_desc prim_desc; - size_t priority = 0UL; + transposeA, + transposeB); }; + if (defaultImplType == impl_desc_type::undef) { + struct PrimitiveDescWithPriority { + dnnl::primitive_desc prim_desc; + size_t priority = 0UL; + }; + + PrimitiveDescWithPriority prim_desc_w_priority{dnnl::primitive_desc(), implPriorities.size()}; + const bool first_match = implPriorities.front() == impl_desc_type::unknown; + + auto cur_desc = createDescriptor(); + + DnnlExtensionUtils::for_each_implementation( + cur_desc, + first_match, + [&](impl_desc_type implType) { + return contains(implPriorities, implType); + }, + [&](dnnl::primitive_desc& desc) { + const impl_desc_type descImplType = parse_impl_name(desc.impl_info_str()); + const auto it = std::find(implPriorities.begin(), implPriorities.end(), descImplType); + const size_t priorityId = std::distance(implPriorities.begin(), it); + const size_t highestPriority = prim_desc_w_priority.priority; + if (priorityId < highestPriority) { + auto desc_copy = dnnl::primitive_desc(DnnlExtensionUtils::clone_primitive_desc(desc.get(true))); + prim_desc_w_priority = {std::move(desc_copy), priorityId}; + } + }); + + return prim_desc_w_priority.prim_desc; + } - PrimitiveDescWithPriority prim_desc_w_priority{dnnl::primitive_desc(), implPriorities.size()}; - const bool first_match = implPriorities.front() == impl_desc_type::unknown; - - auto cur_desc = createDescriptor(); - - DnnlExtensionUtils::for_each_implementation( - cur_desc, - first_match, - [&](impl_desc_type implType) { // is acceptable implementation - return contains(implPriorities, implType); - }, - [&](dnnl::primitive_desc& desc) { // is implementation with highest priority - const impl_desc_type descImplType = parse_impl_name(desc.impl_info_str()); - const auto it = std::find(implPriorities.begin(), implPriorities.end(), descImplType); - const size_t priorityId = std::distance(implPriorities.begin(), it); - const size_t highestPriority = prim_desc_w_priority.priority; - if (priorityId < highestPriority) { - auto desc_copy = dnnl::primitive_desc(DnnlExtensionUtils::clone_primitive_desc(desc.get(true))); - prim_desc_w_priority = {std::move(desc_copy), priorityId}; - } - }); - - return prim_desc_w_priority.prim_desc; - } - - auto prim_desc = createDescriptor(); + auto prim_desc = createDescriptor(); - OPENVINO_ASSERT(prim_desc, "Failed to create matmul primitive descriptor"); - auto first_desc = dnnl::matmul::primitive_desc(prim_desc.get()); + OPENVINO_ASSERT(prim_desc, "Failed to create matmul primitive descriptor"); + auto first_desc = dnnl::matmul::primitive_desc(prim_desc.get()); - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); - if (found) { - return std::move(prim_desc); - } + if (found) { + return std::move(prim_desc); + } - return std::move(first_desc); + return std::move(first_desc); + }); } static std::pair makeDummyInputDims(const Shape& in0, diff --git a/src/plugins/intel_cpu/src/nodes/lrn.cpp b/src/plugins/intel_cpu/src/nodes/lrn.cpp index db8971f4f8b5e3..85ee8b5c42fd61 100644 --- a/src/plugins/intel_cpu/src/nodes/lrn.cpp +++ b/src/plugins/intel_cpu/src/nodes/lrn.cpp @@ -18,6 +18,7 @@ #include #include "common/primitive_hashing_utils.hpp" +#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "graph_context.h" @@ -195,24 +196,26 @@ void Lrn::prepareParams() { auto engine = getEngine(); auto builder = [&engine](const LrnKey& key) -> executorPtr { - auto prim_desc = dnnl::lrn_forward::primitive_desc(engine, - dnnl::prop_kind::forward_inference, - key.alg, - key.inp0->getDnnlDesc(), - key.inp0->getDnnlDesc(), - key.size, - key.alpha, - key.beta, - static_cast(key.k), - key.attr); - - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, key.implType); - - if (!found) { - return nullptr; - } + return withDnnlDescriptorConcurrency([&]() -> executorPtr { + auto prim_desc = dnnl::lrn_forward::primitive_desc(engine, + dnnl::prop_kind::forward_inference, + key.alg, + key.inp0->getDnnlDesc(), + key.inp0->getDnnlDesc(), + key.size, + key.alpha, + key.beta, + static_cast(key.k), + key.attr); + + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, key.implType); + + if (!found) { + return nullptr; + } - return std::make_shared(prim_desc); + return std::make_shared(prim_desc); + }); }; auto cache = context->getParamsCache(); diff --git a/src/plugins/intel_cpu/src/nodes/pooling.cpp b/src/plugins/intel_cpu/src/nodes/pooling.cpp index 8ea9cd948bfb98..cd8fcd5179b351 100644 --- a/src/plugins/intel_cpu/src/nodes/pooling.cpp +++ b/src/plugins/intel_cpu/src/nodes/pooling.cpp @@ -21,6 +21,7 @@ #include "common/primitive_attr.hpp" #include "common/primitive_hashing_utils.hpp" #include "cpu_memory.h" +#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "fake_quantize.h" @@ -536,26 +537,28 @@ void Pooling::prepareParams() { selected_pd->getImplementationType()}; auto engine = getEngine(); auto builder = [&engine](const PoolingKey& key) -> executorPtr { - auto prim_desc = createDescriptorHelper(engine, - key.inp->getDnnlDesc(), - key.out->getDnnlDesc(), - key.alg, - key.stride, - key.kernel, - key.effective_pad_begin, - key.effective_pad_end, - key.effective_dilation, - key.attr); - - auto first_desc = dnnl::pooling_forward::primitive_desc(prim_desc.get()); - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, key.implType); - - if (found) { - return std::make_shared(prim_desc); - } - - // use the first available - return std::make_shared(first_desc); + return withDnnlDescriptorConcurrency([&]() -> executorPtr { + auto prim_desc = createDescriptorHelper(engine, + key.inp->getDnnlDesc(), + key.out->getDnnlDesc(), + key.alg, + key.stride, + key.kernel, + key.effective_pad_begin, + key.effective_pad_end, + key.effective_dilation, + key.attr); + + auto first_desc = dnnl::pooling_forward::primitive_desc(prim_desc.get()); + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, key.implType); + + if (found) { + return std::make_shared(prim_desc); + } + + // use the first available + return std::make_shared(first_desc); + }); }; auto cache = context->getParamsCache(); diff --git a/src/plugins/intel_cpu/src/nodes/rnn.cpp b/src/plugins/intel_cpu/src/nodes/rnn.cpp index 2f39fc86ae59bb..4f8eb3c66638f4 100644 --- a/src/plugins/intel_cpu/src/nodes/rnn.cpp +++ b/src/plugins/intel_cpu/src/nodes/rnn.cpp @@ -21,6 +21,7 @@ #include #include "common/primitive_hashing_utils.hpp" +#include "cpu_parallel.hpp" #include "cpu_memory.h" #include "cpu_types.h" #include "dnnl_extension_utils.h" @@ -1394,16 +1395,18 @@ void RNN::prepareParams() { auto engine = getEngine(); auto builder = [&engine](const RNNKey& key) -> executorPtr { - const auto descPtr = createPrimitiveDescriptor(engine, - key.cellType, - key.cellAct, - key.direction, - key.inDataDescs, - key.outDataDescs, - key.wDescs, - key.attr); - - return descPtr ? std::make_shared(descPtr) : nullptr; + return withDnnlDescriptorConcurrency([&]() -> executorPtr { + const auto descPtr = createPrimitiveDescriptor(engine, + key.cellType, + key.cellAct, + key.direction, + key.inDataDescs, + key.outDataDescs, + key.wDescs, + key.attr); + + return descPtr ? std::make_shared(descPtr) : nullptr; + }); }; auto cache = context->getParamsCache(); diff --git a/src/plugins/intel_cpu/src/nodes/softmax.cpp b/src/plugins/intel_cpu/src/nodes/softmax.cpp index 957d266ecf2a46..30c379c47ffa69 100644 --- a/src/plugins/intel_cpu/src/nodes/softmax.cpp +++ b/src/plugins/intel_cpu/src/nodes/softmax.cpp @@ -17,6 +17,7 @@ #include #include "common/primitive_hashing_utils.hpp" +#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "graph_context.h" @@ -192,35 +193,37 @@ void SoftMax::prepareParams() { auto engine = getEngine(); auto builder = [&engine](const SoftmaxKey& key) -> executorPtr { - auto prim_desc = softmax_forward::primitive_desc(engine, - prop_kind::forward_inference, - algorithm::softmax_accurate, - key.inp0->getDnnlDesc(), - key.inp0->getDnnlDesc(), - key.axis, - key.attr, - true); - - primitive_desc_iterator itpd = prim_desc; - - auto itpd_first = itpd; - while (itpd) { - impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); - if (impl_type == key.implType || - // At least for oneDNN v2.4 the softmax primitive is optimized for the cases where the dimension of the - // softmax axis is physically dense. There could be situations where it is not possible to detect the - // optimized case in advance in case of dynamic shapes, but in runtime the shape could be suitable for - // the optimized implementation, so we have to select the optimized one. - (ref_any == key.implType && (impl_type & jit))) { - prim_desc = itpd.get(); - break; + return withDnnlDescriptorConcurrency([&]() -> executorPtr { + auto prim_desc = softmax_forward::primitive_desc(engine, + prop_kind::forward_inference, + algorithm::softmax_accurate, + key.inp0->getDnnlDesc(), + key.inp0->getDnnlDesc(), + key.axis, + key.attr, + true); + + primitive_desc_iterator itpd = prim_desc; + + auto itpd_first = itpd; + while (itpd) { + impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); + if (impl_type == key.implType || + // At least for oneDNN v2.4 the softmax primitive is optimized for the cases where the dimension of the + // softmax axis is physically dense. There could be situations where it is not possible to detect the + // optimized case in advance in case of dynamic shapes, but in runtime the shape could be suitable for + // the optimized implementation, so we have to select the optimized one. + (ref_any == key.implType && (impl_type & jit))) { + prim_desc = itpd.get(); + break; + } + if (!itpd.next_impl()) { + prim_desc = itpd_first.get(); + break; + } } - if (!itpd.next_impl()) { - prim_desc = itpd_first.get(); - break; - } - } - return std::make_shared(prim_desc); + return std::make_shared(prim_desc); + }); }; auto cache = context->getParamsCache(); From d803949d817f3ffad77f06548917e585630baa24 Mon Sep 17 00:00:00 2001 From: sunxiaoxia2022 Date: Wed, 17 Jun 2026 11:00:10 +0800 Subject: [PATCH 2/7] remove useless function, revert some master logic --- src/plugins/intel_cpu/src/cpu_parallel.hpp | 8 -- src/plugins/intel_cpu/src/nodes/deconv.cpp | 3 +- .../dnnl/dnnl_convolution_primitive.cpp | 99 +++++++++++++------ .../executors/dnnl/dnnl_matmul_primitive.cpp | 4 +- 4 files changed, 70 insertions(+), 44 deletions(-) diff --git a/src/plugins/intel_cpu/src/cpu_parallel.hpp b/src/plugins/intel_cpu/src/cpu_parallel.hpp index d8d1671988bcd1..d5094d04597471 100644 --- a/src/plugins/intel_cpu/src/cpu_parallel.hpp +++ b/src/plugins/intel_cpu/src/cpu_parallel.hpp @@ -40,14 +40,6 @@ class DnnlMaxConcurrencyGuard { bool m_restore_required = false; }; -inline int getDnnlMaxConcurrency() { - int max_concurrency = 0; - if (dnnl_threadpool_interop_get_max_concurrency(&max_concurrency) != dnnl_success) { - return 0; - } - return max_concurrency; -} - template decltype(auto) withDnnlDescriptorConcurrency(F&& f) { DnnlMaxConcurrencyGuard guard(CpuParallel::get_num_worker_threads()); diff --git a/src/plugins/intel_cpu/src/nodes/deconv.cpp b/src/plugins/intel_cpu/src/nodes/deconv.cpp index d557bd97cc8ed5..1f31f0a6c82f6d 100644 --- a/src/plugins/intel_cpu/src/nodes/deconv.cpp +++ b/src/plugins/intel_cpu/src/nodes/deconv.cpp @@ -1047,9 +1047,8 @@ void Deconvolution::prepareParams() { selected_pd->getImplementationType()}; auto engine = getEngine(); - const auto worker_threads = context->getCpuParallel()->get_num_worker_threads(); - auto builder = [&engine, worker_threads](const DeconvKey& key) -> executorPtr { + auto builder = [&engine](const DeconvKey& key) -> executorPtr { dnnl::primitive_desc desc; convolution_forward::primitive_desc fwd_conv_pd; dnnl::memory::desc dnnlBiasDesc; diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp index 20dda6f0e95568..55510183b97902 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp @@ -340,8 +340,41 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, const std::vector& implPriorities, const impl_desc_type defaultImplType) { return withDnnlDescriptorConcurrency([&]() -> primitive_desc { - auto createAnyDescriptor = [&]() -> dnnl::convolution_forward::primitive_desc { - auto inputDescAny = dnnl::memory::desc(inputDesc.get_dims(), inputDesc.get_data_type(), memory::format_tag::any); + auto createPrimitiveDescriptor = [&](const dnnl::primitive_attr& attr) { + return fcSemantic ? createInnerProductDescriptor(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine) + : createConvolutionDescriptor(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); + }; + + auto createConvolutionDescriptorAny = [](const dnnl::memory::desc& inputDesc, + const dnnl::memory::desc& weightDesc, + const dnnl::memory::desc& biasDesc, + const dnnl::memory::desc& outputDesc, + const std::vector& stride, + const std::vector& dilation, + const std::vector& paddingL, + const std::vector& paddingR, + const dnnl::primitive_attr& attr, + const dnnl::engine& engine) { + auto inputDescAny = + dnnl::memory::desc(inputDesc.get_dims(), inputDesc.get_data_type(), memory::format_tag::any); auto outputDescAny = dnnl::memory::desc(outputDesc.get_dims(), outputDesc.get_data_type(), memory::format_tag::any); return createConvolutionDescriptor(inputDescAny, @@ -356,59 +389,61 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, engine); }; - auto createSelectedDescriptor = [&]() -> dnnl::convolution_forward::primitive_desc { - if (fcSemantic) { - return createInnerProductDescriptor(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - } - - return createConvolutionDescriptor(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - }; - - auto prim_desc = createSelectedDescriptor(); + auto prim_desc = createPrimitiveDescriptor(attr); + // keep first implementation descriptor to fallback to it if no other implementation is found auto first_desc = prim_desc; + // if default implementation type is not specified, try to find the best implementation if (defaultImplType == impl_desc_type::undef) { if (!prim_desc) { - return createAnyDescriptor(); + // fallback to 'any' implementation + return createConvolutionDescriptorAny(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); } for (auto preferredImplType : implPriorities) { + // the only way to fully reset primitive_desc after iterating over the implementations is to re-create it const bool found = DnnlExtensionUtils::find_implementation(prim_desc, preferredImplType); if (found) { return std::move(prim_desc); } - prim_desc = createSelectedDescriptor(); + prim_desc = createPrimitiveDescriptor(attr); } return std::move(first_desc); } + // try to use a default implementations type (created using dummy shapes) if specified const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); if (found) { return std::move(prim_desc); } - return std::move(first_desc); + if (fcSemantic) { // fallback to the first implementation if used as FC executor + return std::move(first_desc); + } + + // fallback to 'any' implementation + return createConvolutionDescriptorAny(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); }); } diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp index cdb1c5cb93a9cf..a27443f72d760b 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp @@ -370,10 +370,10 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, DnnlExtensionUtils::for_each_implementation( cur_desc, first_match, - [&](impl_desc_type implType) { + [&](impl_desc_type implType) { // is acceptable implementation return contains(implPriorities, implType); }, - [&](dnnl::primitive_desc& desc) { + [&](dnnl::primitive_desc& desc) { // is implementation with highest priority const impl_desc_type descImplType = parse_impl_name(desc.impl_info_str()); const auto it = std::find(implPriorities.begin(), implPriorities.end(), descImplType); const size_t priorityId = std::distance(implPriorities.begin(), it); From f11e72cff30e19928602798210394096388b94df Mon Sep 17 00:00:00 2001 From: sunxiaoxia2022 Date: Wed, 17 Jun 2026 13:59:16 +0800 Subject: [PATCH 3/7] add withDnnlDescriptorConcurrency for createDescriptor in gathermatmul --- .../dnnl/dnnl_gathermatmul_executor.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_gathermatmul_executor.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_gathermatmul_executor.cpp index 68bc9488dece93..6d0a37704256ef 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_gathermatmul_executor.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_gathermatmul_executor.cpp @@ -25,6 +25,7 @@ #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) # include #endif +#include "cpu_parallel.hpp" #include "cpu_memory.h" #include "cpu_types.h" #include "dnnl_extension_utils.h" @@ -116,13 +117,15 @@ class GatherMatmulDnnlExecutor::InnerProduct { const auto& bias_md = key.bias_md; - auto ip_prim_desc = dnnl::inner_product_forward::primitive_desc(eng, - dnnl::prop_kind::forward_inference, - m_input_md, - weights_md, - bias_md, - m_output_md, - m_attr); + auto ip_prim_desc = withDnnlDescriptorConcurrency([&]() { + return dnnl::inner_product_forward::primitive_desc(eng, + dnnl::prop_kind::forward_inference, + m_input_md, + weights_md, + bias_md, + m_output_md, + m_attr); + }); m_impl_type = parse_impl_name(ip_prim_desc.impl_info_str()); m_wei_md = ip_prim_desc.weights_desc(); From 950eb78a5de8775c50471f6ebed54e9adeec4c8b Mon Sep 17 00:00:00 2001 From: sunxiaoxia2022 Date: Thu, 2 Jul 2026 13:41:32 +0800 Subject: [PATCH 4/7] revert current change --- src/plugins/intel_cpu/src/cpu_parallel.hpp | 47 +---- src/plugins/intel_cpu/src/graph.cpp | 6 +- src/plugins/intel_cpu/src/nodes/deconv.cpp | 37 ++-- .../dnnl/dnnl_convolution_primitive.cpp | 190 +++++++++--------- .../dnnl/dnnl_fullyconnected_primitive.cpp | 39 ++-- .../dnnl/dnnl_gathermatmul_executor.cpp | 17 +- .../executors/dnnl/dnnl_matmul_primitive.cpp | 101 +++++----- src/plugins/intel_cpu/src/nodes/lrn.cpp | 37 ++-- src/plugins/intel_cpu/src/nodes/pooling.cpp | 43 ++-- src/plugins/intel_cpu/src/nodes/rnn.cpp | 23 +-- src/plugins/intel_cpu/src/nodes/softmax.cpp | 59 +++--- 11 files changed, 262 insertions(+), 337 deletions(-) diff --git a/src/plugins/intel_cpu/src/cpu_parallel.hpp b/src/plugins/intel_cpu/src/cpu_parallel.hpp index d5094d04597471..1deee945c4f2d9 100644 --- a/src/plugins/intel_cpu/src/cpu_parallel.hpp +++ b/src/plugins/intel_cpu/src/cpu_parallel.hpp @@ -12,46 +12,6 @@ namespace ov::intel_cpu { class ThreadPool; -#if OV_THREAD == OV_THREAD_TBB_ADAPTIVE -class DnnlMaxConcurrencyGuard { -public: - explicit DnnlMaxConcurrencyGuard(int max_concurrency) { - if (dnnl_threadpool_interop_get_max_concurrency(&m_prev_max_concurrency) != dnnl_success) { - return; - } - - if (m_prev_max_concurrency == max_concurrency) { - return; - } - - if (dnnl_threadpool_interop_set_max_concurrency(max_concurrency) == dnnl_success) { - m_restore_required = true; - } - } - - ~DnnlMaxConcurrencyGuard() { - if (m_restore_required) { - dnnl_threadpool_interop_set_max_concurrency(m_prev_max_concurrency); - } - } - -private: - int m_prev_max_concurrency = 0; - bool m_restore_required = false; -}; - -template -decltype(auto) withDnnlDescriptorConcurrency(F&& f) { - DnnlMaxConcurrencyGuard guard(CpuParallel::get_num_worker_threads()); - return std::forward(f)(); -} -#else -template -decltype(auto) withDnnlDescriptorConcurrency(F&& f) { - return std::forward(f)(); -} -#endif - class CpuParallel { public: // Default multiplier for the number of virtual threads when tbb partitioner is AUTO. This value is determined @@ -82,12 +42,7 @@ class CpuParallel { [[nodiscard]] static int get_num_worker_threads() { return parallel_get_max_threads(); } - void activateForInit() const { -#if OV_THREAD == OV_THREAD_TBB_ADAPTIVE - dnnl_threadpool_interop_set_max_concurrency(get_num_worker_threads()); -#endif - } - void activateForExecution() const { + void activate() const { #if OV_THREAD == OV_THREAD_TBB_ADAPTIVE dnnl_threadpool_interop_set_max_concurrency(get_num_threads()); #endif diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index d81f81118569e5..0853daf9b3afd3 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -128,7 +128,7 @@ void Graph::Init(const std::vector& graphNodes, m_context = context; m_stream = make_stream(getEngine(), m_context->getCpuParallel()->get_thread_pool()); - m_context->getCpuParallel()->activateForInit(); + m_context->getCpuParallel()->activate(); this->_name = std::move(name); @@ -385,7 +385,7 @@ void Graph::Init(const std::shared_ptr& model, m_context = context; m_stream = make_stream(getEngine(), m_context->getCpuParallel()->get_thread_pool()); - m_context->getCpuParallel()->activateForInit(); + m_context->getCpuParallel()->activate(); Replicate(model, inputConfigs, outputConfigs); @@ -397,8 +397,6 @@ void Graph::Activate() { // the allocation context collection from the outer graph so the state for inner graph is "Ready" // We probably want to avoid such uncertainty // OPENVINO_ASSERT(status == Status::Initialized, "Invalid graph status: ", static_cast(status)); - m_context->getCpuParallel()->activateForExecution(); - Allocate(); CreatePrimitivesAndExecConstants(); diff --git a/src/plugins/intel_cpu/src/nodes/deconv.cpp b/src/plugins/intel_cpu/src/nodes/deconv.cpp index 1f31f0a6c82f6d..07eea910122f06 100644 --- a/src/plugins/intel_cpu/src/nodes/deconv.cpp +++ b/src/plugins/intel_cpu/src/nodes/deconv.cpp @@ -24,7 +24,6 @@ #include "common/primitive_hashing_utils.hpp" #include "cpu/x64/cpu_isa_traits.hpp" #include "cpu_memory.h" -#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "dnnl_postops_composer_legacy.h" @@ -830,37 +829,35 @@ dnnl::primitive_desc createDescriptorInternal(const dnnl::memory::desc& in_candi const ov::CoordinateDiff& paddingR, const dnnl::primitive_attr& attr, const dnnl::engine& engine) { - return withDnnlDescriptorConcurrency([&]() -> dnnl::primitive_desc { - auto convertDims = [](const std::vector& orig_dims) { - return memory::dims(orig_dims.begin(), orig_dims.end()); - }; + auto convertDims = [](const std::vector& orig_dims) { + return memory::dims(orig_dims.begin(), orig_dims.end()); + }; - if (with_bias) { - return dnnl::deconvolution_forward::primitive_desc(engine, - prop_kind::forward_inference, - dnnl::algorithm::deconvolution_direct, - in_candidate, - wgh_candidate, - bias_candidate, - out_candidate, - convertDims(stride), - convertDims(dilation), - convertDims(paddingL), - convertDims(paddingR), - attr); - } + if (with_bias) { return dnnl::deconvolution_forward::primitive_desc(engine, prop_kind::forward_inference, dnnl::algorithm::deconvolution_direct, in_candidate, wgh_candidate, + bias_candidate, out_candidate, convertDims(stride), convertDims(dilation), convertDims(paddingL), convertDims(paddingR), attr); - }); + } + return dnnl::deconvolution_forward::primitive_desc(engine, + prop_kind::forward_inference, + dnnl::algorithm::deconvolution_direct, + in_candidate, + wgh_candidate, + out_candidate, + convertDims(stride), + convertDims(dilation), + convertDims(paddingL), + convertDims(paddingR), + attr); } } // namespace diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp index 55510183b97902..a990b2c7f23b7e 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_convolution_primitive.cpp @@ -24,7 +24,6 @@ #include #include -#include "cpu_parallel.hpp" #include "cpu/x64/cpu_isa_traits.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" @@ -339,112 +338,109 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, bool fcSemantic, const std::vector& implPriorities, const impl_desc_type defaultImplType) { - return withDnnlDescriptorConcurrency([&]() -> primitive_desc { - auto createPrimitiveDescriptor = [&](const dnnl::primitive_attr& attr) { - return fcSemantic ? createInnerProductDescriptor(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine) - : createConvolutionDescriptor(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - }; - - auto createConvolutionDescriptorAny = [](const dnnl::memory::desc& inputDesc, - const dnnl::memory::desc& weightDesc, - const dnnl::memory::desc& biasDesc, - const dnnl::memory::desc& outputDesc, - const std::vector& stride, - const std::vector& dilation, - const std::vector& paddingL, - const std::vector& paddingR, - const dnnl::primitive_attr& attr, - const dnnl::engine& engine) { - auto inputDescAny = - dnnl::memory::desc(inputDesc.get_dims(), inputDesc.get_data_type(), memory::format_tag::any); - auto outputDescAny = - dnnl::memory::desc(outputDesc.get_dims(), outputDesc.get_data_type(), memory::format_tag::any); - return createConvolutionDescriptor(inputDescAny, - weightDesc, - biasDesc, - outputDescAny, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - }; - - auto prim_desc = createPrimitiveDescriptor(attr); - // keep first implementation descriptor to fallback to it if no other implementation is found - auto first_desc = prim_desc; - // if default implementation type is not specified, try to find the best implementation - if (defaultImplType == impl_desc_type::undef) { - if (!prim_desc) { - // fallback to 'any' implementation - return createConvolutionDescriptorAny(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - } + auto createPrimitiveDescriptor = [&](const dnnl::primitive_attr& attr) { + return fcSemantic ? createInnerProductDescriptor(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine) + : createConvolutionDescriptor(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); + }; - for (auto preferredImplType : implPriorities) { - // the only way to fully reset primitive_desc after iterating over the implementations is to re-create it - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, preferredImplType); + auto createConvolutionDescriptorAny = [](const dnnl::memory::desc& inputDesc, + const dnnl::memory::desc& weightDesc, + const dnnl::memory::desc& biasDesc, + const dnnl::memory::desc& outputDesc, + const std::vector& stride, + const std::vector& dilation, + const std::vector& paddingL, + const std::vector& paddingR, + const dnnl::primitive_attr& attr, + const dnnl::engine& engine) { + auto inputDescAny = + dnnl::memory::desc(inputDesc.get_dims(), inputDesc.get_data_type(), memory::format_tag::any); + auto outputDescAny = + dnnl::memory::desc(outputDesc.get_dims(), outputDesc.get_data_type(), memory::format_tag::any); + return createConvolutionDescriptor(inputDescAny, + weightDesc, + biasDesc, + outputDescAny, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); + }; - if (found) { - return std::move(prim_desc); - } + auto prim_desc = createPrimitiveDescriptor(attr); + // keep first implementation descriptor to fallback to it if no other implementation is found + auto first_desc = prim_desc; + // if default implementation type is not specified, try to find the best implementation + if (defaultImplType == impl_desc_type::undef) { + if (!prim_desc) { + // fallback to 'any' implementation + return createConvolutionDescriptorAny(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); + } + + for (auto preferredImplType : implPriorities) { + // the only way to fully reset primitive_desc after iterating over the implementations is to re-create it + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, preferredImplType); - prim_desc = createPrimitiveDescriptor(attr); + if (found) { + return std::move(prim_desc); } - return std::move(first_desc); + prim_desc = createPrimitiveDescriptor(attr); } - // try to use a default implementations type (created using dummy shapes) if specified - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); + return std::move(first_desc); + } + // try to use a default implementations type (created using dummy shapes) if specified + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); - if (found) { - return std::move(prim_desc); - } + if (found) { + return std::move(prim_desc); + } - if (fcSemantic) { // fallback to the first implementation if used as FC executor - return std::move(first_desc); - } + if (fcSemantic) { // fallback to the first implementation if used as FC executor + return std::move(first_desc); + } - // fallback to 'any' implementation - return createConvolutionDescriptorAny(inputDesc, - weightDesc, - biasDesc, - outputDesc, - stride, - dilation, - paddingL, - paddingR, - attr, - engine); - }); + // fallback to 'any' implementation + return createConvolutionDescriptorAny(inputDesc, + weightDesc, + biasDesc, + outputDesc, + stride, + dilation, + paddingL, + paddingR, + attr, + engine); } static std::vector createPrimitiveAttrs(const ConvAttrs& attrs, diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp index 911b809b5fee4e..65d4473cbebf0f 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_fullyconnected_primitive.cpp @@ -19,7 +19,6 @@ #include #include "config.h" -#include "cpu_parallel.hpp" #include "cpu/x64/cpu_isa_traits.hpp" #include "cpu_memory.h" #include "cpu_types.h" @@ -367,28 +366,26 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, const std::vector& implPriorities, const bool useSparseWeights, const bool useWeightsDecompression) { - return withDnnlDescriptorConcurrency([&]() -> primitive_desc { - auto prim_desc = createDescriptorInternal(inputDesc, - weightDesc, - biasDesc, - outputDesc, - attr, - engine, - useSparseWeights, - useWeightsDecompression); - OPENVINO_ASSERT(prim_desc, "Failed to create inner_product primitive descriptor"); - auto first_desc = dnnl::inner_product_forward::primitive_desc(prim_desc.get()); - - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, [&](impl_desc_type implType) { - return contains(implPriorities, implType); - }); - - if (found) { - return std::move(prim_desc); - } + auto prim_desc = createDescriptorInternal(inputDesc, + weightDesc, + biasDesc, + outputDesc, + attr, + engine, + useSparseWeights, + useWeightsDecompression); + OPENVINO_ASSERT(prim_desc, "Failed to create inner_product primitive descriptor"); + auto first_desc = dnnl::inner_product_forward::primitive_desc(prim_desc.get()); - return std::move(first_desc); + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, [&](impl_desc_type implType) { + return contains(implPriorities, implType); }); + + if (found) { + return std::move(prim_desc); + } + + return std::move(first_desc); } static VectorDims makeDummyInputDims(const Shape& inShape, const Shape& wShape) { diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_gathermatmul_executor.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_gathermatmul_executor.cpp index 6d0a37704256ef..68bc9488dece93 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_gathermatmul_executor.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_gathermatmul_executor.cpp @@ -25,7 +25,6 @@ #if defined(OPENVINO_ARCH_X86) || defined(OPENVINO_ARCH_X86_64) # include #endif -#include "cpu_parallel.hpp" #include "cpu_memory.h" #include "cpu_types.h" #include "dnnl_extension_utils.h" @@ -117,15 +116,13 @@ class GatherMatmulDnnlExecutor::InnerProduct { const auto& bias_md = key.bias_md; - auto ip_prim_desc = withDnnlDescriptorConcurrency([&]() { - return dnnl::inner_product_forward::primitive_desc(eng, - dnnl::prop_kind::forward_inference, - m_input_md, - weights_md, - bias_md, - m_output_md, - m_attr); - }); + auto ip_prim_desc = dnnl::inner_product_forward::primitive_desc(eng, + dnnl::prop_kind::forward_inference, + m_input_md, + weights_md, + bias_md, + m_output_md, + m_attr); m_impl_type = parse_impl_name(ip_prim_desc.impl_info_str()); m_wei_md = ip_prim_desc.weights_desc(); diff --git a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp index a27443f72d760b..f45604af0971c8 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/dnnl/dnnl_matmul_primitive.cpp @@ -20,7 +20,6 @@ #include #include "cpu_memory.h" -#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "dnnl_postops_composer.h" @@ -338,68 +337,66 @@ static primitive_desc createPrimitiveDesc(const dnnl::memory::desc& inputDesc, [[maybe_unused]] const bool useSparseWeights, const bool useWeightsDecompression, const bool fcSemantic) { - return withDnnlDescriptorConcurrency([&]() -> primitive_desc { - auto createDescriptor = [&]() { - return fcSemantic ? createDescriptorInternalAsFc(inputDesc, - weightDesc, - biasDesc, - outputDesc, - attr, - engine, - useWeightsDecompression) - : createDescriptorInternal(inputDesc, + auto createDescriptor = [&]() { + return fcSemantic ? createDescriptorInternalAsFc(inputDesc, weightDesc, biasDesc, outputDesc, attr, engine, - transposeA, - transposeB); + useWeightsDecompression) + : createDescriptorInternal(inputDesc, + weightDesc, + biasDesc, + outputDesc, + attr, + engine, + transposeA, + transposeB); + }; + if (defaultImplType == impl_desc_type::undef) { + struct PrimitiveDescWithPriority { + dnnl::primitive_desc prim_desc; + size_t priority = 0UL; }; - if (defaultImplType == impl_desc_type::undef) { - struct PrimitiveDescWithPriority { - dnnl::primitive_desc prim_desc; - size_t priority = 0UL; - }; - - PrimitiveDescWithPriority prim_desc_w_priority{dnnl::primitive_desc(), implPriorities.size()}; - const bool first_match = implPriorities.front() == impl_desc_type::unknown; - - auto cur_desc = createDescriptor(); - - DnnlExtensionUtils::for_each_implementation( - cur_desc, - first_match, - [&](impl_desc_type implType) { // is acceptable implementation - return contains(implPriorities, implType); - }, - [&](dnnl::primitive_desc& desc) { // is implementation with highest priority - const impl_desc_type descImplType = parse_impl_name(desc.impl_info_str()); - const auto it = std::find(implPriorities.begin(), implPriorities.end(), descImplType); - const size_t priorityId = std::distance(implPriorities.begin(), it); - const size_t highestPriority = prim_desc_w_priority.priority; - if (priorityId < highestPriority) { - auto desc_copy = dnnl::primitive_desc(DnnlExtensionUtils::clone_primitive_desc(desc.get(true))); - prim_desc_w_priority = {std::move(desc_copy), priorityId}; - } - }); - - return prim_desc_w_priority.prim_desc; - } - auto prim_desc = createDescriptor(); + PrimitiveDescWithPriority prim_desc_w_priority{dnnl::primitive_desc(), implPriorities.size()}; + const bool first_match = implPriorities.front() == impl_desc_type::unknown; + + auto cur_desc = createDescriptor(); + + DnnlExtensionUtils::for_each_implementation( + cur_desc, + first_match, + [&](impl_desc_type implType) { // is acceptable implementation + return contains(implPriorities, implType); + }, + [&](dnnl::primitive_desc& desc) { // is implementation with highest priority + const impl_desc_type descImplType = parse_impl_name(desc.impl_info_str()); + const auto it = std::find(implPriorities.begin(), implPriorities.end(), descImplType); + const size_t priorityId = std::distance(implPriorities.begin(), it); + const size_t highestPriority = prim_desc_w_priority.priority; + if (priorityId < highestPriority) { + auto desc_copy = dnnl::primitive_desc(DnnlExtensionUtils::clone_primitive_desc(desc.get(true))); + prim_desc_w_priority = {std::move(desc_copy), priorityId}; + } + }); + + return prim_desc_w_priority.prim_desc; + } - OPENVINO_ASSERT(prim_desc, "Failed to create matmul primitive descriptor"); - auto first_desc = dnnl::matmul::primitive_desc(prim_desc.get()); + auto prim_desc = createDescriptor(); - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); + OPENVINO_ASSERT(prim_desc, "Failed to create matmul primitive descriptor"); + auto first_desc = dnnl::matmul::primitive_desc(prim_desc.get()); - if (found) { - return std::move(prim_desc); - } + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, defaultImplType); + + if (found) { + return std::move(prim_desc); + } - return std::move(first_desc); - }); + return std::move(first_desc); } static std::pair makeDummyInputDims(const Shape& in0, diff --git a/src/plugins/intel_cpu/src/nodes/lrn.cpp b/src/plugins/intel_cpu/src/nodes/lrn.cpp index c78a23838c354b..ba1256a9fa94af 100644 --- a/src/plugins/intel_cpu/src/nodes/lrn.cpp +++ b/src/plugins/intel_cpu/src/nodes/lrn.cpp @@ -18,7 +18,6 @@ #include #include "common/primitive_hashing_utils.hpp" -#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "graph_context.h" @@ -198,26 +197,24 @@ void Lrn::prepareParams() { auto engine = getEngine(); auto builder = [&engine](const LrnKey& key) -> executorPtr { - return withDnnlDescriptorConcurrency([&]() -> executorPtr { - auto prim_desc = dnnl::lrn_forward::primitive_desc(engine, - dnnl::prop_kind::forward_inference, - key.alg, - key.inp0->getDnnlDesc(), - key.inp0->getDnnlDesc(), - key.size, - key.alpha, - key.beta, - static_cast(key.k), - key.attr); - - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, key.implType); - - if (!found) { - return nullptr; - } + auto prim_desc = dnnl::lrn_forward::primitive_desc(engine, + dnnl::prop_kind::forward_inference, + key.alg, + key.inp0->getDnnlDesc(), + key.inp0->getDnnlDesc(), + key.size, + key.alpha, + key.beta, + static_cast(key.k), + key.attr); + + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, key.implType); + + if (!found) { + return nullptr; + } - return std::make_shared(prim_desc); - }); + return std::make_shared(prim_desc); }; auto cache = context->getParamsCache(); diff --git a/src/plugins/intel_cpu/src/nodes/pooling.cpp b/src/plugins/intel_cpu/src/nodes/pooling.cpp index cd8fcd5179b351..8ea9cd948bfb98 100644 --- a/src/plugins/intel_cpu/src/nodes/pooling.cpp +++ b/src/plugins/intel_cpu/src/nodes/pooling.cpp @@ -21,7 +21,6 @@ #include "common/primitive_attr.hpp" #include "common/primitive_hashing_utils.hpp" #include "cpu_memory.h" -#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "fake_quantize.h" @@ -537,28 +536,26 @@ void Pooling::prepareParams() { selected_pd->getImplementationType()}; auto engine = getEngine(); auto builder = [&engine](const PoolingKey& key) -> executorPtr { - return withDnnlDescriptorConcurrency([&]() -> executorPtr { - auto prim_desc = createDescriptorHelper(engine, - key.inp->getDnnlDesc(), - key.out->getDnnlDesc(), - key.alg, - key.stride, - key.kernel, - key.effective_pad_begin, - key.effective_pad_end, - key.effective_dilation, - key.attr); - - auto first_desc = dnnl::pooling_forward::primitive_desc(prim_desc.get()); - const bool found = DnnlExtensionUtils::find_implementation(prim_desc, key.implType); - - if (found) { - return std::make_shared(prim_desc); - } - - // use the first available - return std::make_shared(first_desc); - }); + auto prim_desc = createDescriptorHelper(engine, + key.inp->getDnnlDesc(), + key.out->getDnnlDesc(), + key.alg, + key.stride, + key.kernel, + key.effective_pad_begin, + key.effective_pad_end, + key.effective_dilation, + key.attr); + + auto first_desc = dnnl::pooling_forward::primitive_desc(prim_desc.get()); + const bool found = DnnlExtensionUtils::find_implementation(prim_desc, key.implType); + + if (found) { + return std::make_shared(prim_desc); + } + + // use the first available + return std::make_shared(first_desc); }; auto cache = context->getParamsCache(); diff --git a/src/plugins/intel_cpu/src/nodes/rnn.cpp b/src/plugins/intel_cpu/src/nodes/rnn.cpp index 4f8eb3c66638f4..2f39fc86ae59bb 100644 --- a/src/plugins/intel_cpu/src/nodes/rnn.cpp +++ b/src/plugins/intel_cpu/src/nodes/rnn.cpp @@ -21,7 +21,6 @@ #include #include "common/primitive_hashing_utils.hpp" -#include "cpu_parallel.hpp" #include "cpu_memory.h" #include "cpu_types.h" #include "dnnl_extension_utils.h" @@ -1395,18 +1394,16 @@ void RNN::prepareParams() { auto engine = getEngine(); auto builder = [&engine](const RNNKey& key) -> executorPtr { - return withDnnlDescriptorConcurrency([&]() -> executorPtr { - const auto descPtr = createPrimitiveDescriptor(engine, - key.cellType, - key.cellAct, - key.direction, - key.inDataDescs, - key.outDataDescs, - key.wDescs, - key.attr); - - return descPtr ? std::make_shared(descPtr) : nullptr; - }); + const auto descPtr = createPrimitiveDescriptor(engine, + key.cellType, + key.cellAct, + key.direction, + key.inDataDescs, + key.outDataDescs, + key.wDescs, + key.attr); + + return descPtr ? std::make_shared(descPtr) : nullptr; }; auto cache = context->getParamsCache(); diff --git a/src/plugins/intel_cpu/src/nodes/softmax.cpp b/src/plugins/intel_cpu/src/nodes/softmax.cpp index b46775ec106c98..bbc9ed8a369c2f 100644 --- a/src/plugins/intel_cpu/src/nodes/softmax.cpp +++ b/src/plugins/intel_cpu/src/nodes/softmax.cpp @@ -17,7 +17,6 @@ #include #include "common/primitive_hashing_utils.hpp" -#include "cpu_parallel.hpp" #include "cpu_types.h" #include "dnnl_extension_utils.h" #include "graph_context.h" @@ -195,37 +194,35 @@ void SoftMax::prepareParams() { auto engine = getEngine(); auto builder = [&engine](const SoftmaxKey& key) -> executorPtr { - return withDnnlDescriptorConcurrency([&]() -> executorPtr { - auto prim_desc = softmax_forward::primitive_desc(engine, - prop_kind::forward_inference, - algorithm::softmax_accurate, - key.inp0->getDnnlDesc(), - key.inp0->getDnnlDesc(), - key.axis, - key.attr, - true); - - primitive_desc_iterator itpd = prim_desc; - - auto itpd_first = itpd; - while (itpd) { - impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); - if (impl_type == key.implType || - // At least for oneDNN v2.4 the softmax primitive is optimized for the cases where the dimension of the - // softmax axis is physically dense. There could be situations where it is not possible to detect the - // optimized case in advance in case of dynamic shapes, but in runtime the shape could be suitable for - // the optimized implementation, so we have to select the optimized one. - (ref_any == key.implType && (impl_type & jit))) { - prim_desc = itpd.get(); - break; - } - if (!itpd.next_impl()) { - prim_desc = itpd_first.get(); - break; - } + auto prim_desc = softmax_forward::primitive_desc(engine, + prop_kind::forward_inference, + algorithm::softmax_accurate, + key.inp0->getDnnlDesc(), + key.inp0->getDnnlDesc(), + key.axis, + key.attr, + true); + + primitive_desc_iterator itpd = prim_desc; + + auto itpd_first = itpd; + while (itpd) { + impl_desc_type impl_type = parse_impl_name(itpd.impl_info_str()); + if (impl_type == key.implType || + // At least for oneDNN v2.4 the softmax primitive is optimized for the cases where the dimension of the + // softmax axis is physically dense. There could be situations where it is not possible to detect the + // optimized case in advance in case of dynamic shapes, but in runtime the shape could be suitable for + // the optimized implementation, so we have to select the optimized one. + (ref_any == key.implType && (impl_type & jit))) { + prim_desc = itpd.get(); + break; } - return std::make_shared(prim_desc); - }); + if (!itpd.next_impl()) { + prim_desc = itpd_first.get(); + break; + } + } + return std::make_shared(prim_desc); }; auto cache = context->getParamsCache(); From b7d5d99a38b072ec7e2830275b51406f38d8b18c Mon Sep 17 00:00:00 2001 From: sunxiaoxia2022 Date: Thu, 2 Jul 2026 13:45:53 +0800 Subject: [PATCH 5/7] revert activate --- src/plugins/intel_cpu/src/node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_cpu/src/node.cpp b/src/plugins/intel_cpu/src/node.cpp index 3f2f7342dd1a55..1090167c93df98 100644 --- a/src/plugins/intel_cpu/src/node.cpp +++ b/src/plugins/intel_cpu/src/node.cpp @@ -813,7 +813,7 @@ void Node::updateDynamicParams() { getName(), " ", getOriginalLayers()); - context->getCpuParallel()->activateForExecution(); + context->getCpuParallel()->activate(); prepareParams(); } } From aed6d7d0e36a1f3cf678c87cd5a0e6e5f76b56f8 Mon Sep 17 00:00:00 2001 From: sunxiaoxia2022 Date: Fri, 3 Jul 2026 09:20:50 +0800 Subject: [PATCH 6/7] fix oom of scratchpad --- src/plugins/intel_cpu/src/cpu_parallel.hpp | 1 + src/plugins/intel_cpu/thirdparty/onednn | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/intel_cpu/src/cpu_parallel.hpp b/src/plugins/intel_cpu/src/cpu_parallel.hpp index 1deee945c4f2d9..01346c0554c037 100644 --- a/src/plugins/intel_cpu/src/cpu_parallel.hpp +++ b/src/plugins/intel_cpu/src/cpu_parallel.hpp @@ -45,6 +45,7 @@ class CpuParallel { void activate() const { #if OV_THREAD == OV_THREAD_TBB_ADAPTIVE dnnl_threadpool_interop_set_max_concurrency(get_num_threads()); + dnnl_threadpool_interop_set_scratchpad_concurrency(get_num_worker_threads()); #endif } diff --git a/src/plugins/intel_cpu/thirdparty/onednn b/src/plugins/intel_cpu/thirdparty/onednn index f82d833de6f13f..56b6481f8d4517 160000 --- a/src/plugins/intel_cpu/thirdparty/onednn +++ b/src/plugins/intel_cpu/thirdparty/onednn @@ -1 +1 @@ -Subproject commit f82d833de6f13fac4bb1926d521ca8fec4f4ae01 +Subproject commit 56b6481f8d4517d298cb1d0dcdb125ce784b12d0 From 384386964fcc974b79c2198edd9eea038806ed90 Mon Sep 17 00:00:00 2001 From: sunxiaoxia2022 Date: Thu, 16 Jul 2026 21:25:15 +0800 Subject: [PATCH 7/7] add comments for dnnl_threadpool_interop_set_scratchpad_concurrency --- src/plugins/intel_cpu/src/cpu_parallel.hpp | 3 +++ src/plugins/intel_cpu/thirdparty/onednn | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/intel_cpu/src/cpu_parallel.hpp b/src/plugins/intel_cpu/src/cpu_parallel.hpp index 01346c0554c037..515e421f4c6dfe 100644 --- a/src/plugins/intel_cpu/src/cpu_parallel.hpp +++ b/src/plugins/intel_cpu/src/cpu_parallel.hpp @@ -45,6 +45,9 @@ class CpuParallel { void activate() const { #if OV_THREAD == OV_THREAD_TBB_ADAPTIVE dnnl_threadpool_interop_set_max_concurrency(get_num_threads()); + // oneDNN uses this value when estimating scratchpad outside an active parallel region. + // Keep it aligned with the real worker pool size: AUTO may raise max_concurrency via + // virtual threads for scheduling, but scratchpad is still needed only for worker threads. dnnl_threadpool_interop_set_scratchpad_concurrency(get_num_worker_threads()); #endif } diff --git a/src/plugins/intel_cpu/thirdparty/onednn b/src/plugins/intel_cpu/thirdparty/onednn index 56b6481f8d4517..53174cdae98e98 160000 --- a/src/plugins/intel_cpu/thirdparty/onednn +++ b/src/plugins/intel_cpu/thirdparty/onednn @@ -1 +1 @@ -Subproject commit 56b6481f8d4517d298cb1d0dcdb125ce784b12d0 +Subproject commit 53174cdae98e984c185dec6541699c02f34dbad7