From 1c3c3523909322193f50f96bc9d8958418c24914 Mon Sep 17 00:00:00 2001 From: mangguo Date: Thu, 6 Aug 2026 16:07:43 +0800 Subject: [PATCH 1/8] Add DisableBF16CompCumSumSinGen transformation to keep l_sin_gen CumSum chain in FP32 --- src/plugins/intel_cpu/src/graph.cpp | 10 + src/plugins/intel_cpu/src/node.cpp | 1 + src/plugins/intel_cpu/src/node.h | 4 + .../pass/disable_bf16_comp_cumsum_sin_gen.cpp | 101 +++++++++ .../pass/disable_bf16_comp_cumsum_sin_gen.hpp | 38 ++++ .../transformation_pipeline.cpp | 2 + .../disable_bf16_comp_cumsum_sin_gen_test.cpp | 203 ++++++++++++++++++ 7 files changed, 359 insertions(+) create mode 100644 src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp create mode 100644 src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp create mode 100644 src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index 5bb6a7ab4401f7..f0b336d46f5ccd 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -2092,6 +2092,16 @@ void Graph::EnforceInferencePrecision() const { // Pattern-based node skipping for BF16 precision enforcement if (inferPrec == ov::element::bf16) { + // Preserve a continuous f32 region around model-declared precision-sensitive operations. + for (const auto& node : graphNodes) { + if (!node->isBF16ConversionDisabled()) { + continue; + } + nodesToSkip.insert(node); + backwardSkipSearch(node, nodesToSkip); + forwardSkipSearch(node, nodesToSkip); + } + for (const auto& node : graphNodes) { // Pattern 1: MatMul with Convert from integer to floating point on any input. This basically means that // converting such an integer input to bf16 leads to loosing accuracy, as bf16 can only exactly represent diff --git a/src/plugins/intel_cpu/src/node.cpp b/src/plugins/intel_cpu/src/node.cpp index 15be8a84cfb992..39651590d5da0f 100644 --- a/src/plugins/intel_cpu/src/node.cpp +++ b/src/plugins/intel_cpu/src/node.cpp @@ -190,6 +190,7 @@ Node::Node(const std::shared_ptr& op, GraphContext::CPtr ctx, const Sh if (is_conversion_disabled(op, element::f16)) { keepOriginalPrecision = true; } + disableBF16Conversion = is_conversion_disabled(op, element::f32, element::bf16); } Node::Node(const std::string& type, diff --git a/src/plugins/intel_cpu/src/node.h b/src/plugins/intel_cpu/src/node.h index b867ea31a8e974..f2ef5cc70d0482 100644 --- a/src/plugins/intel_cpu/src/node.h +++ b/src/plugins/intel_cpu/src/node.h @@ -690,6 +690,9 @@ class Node { bool keepOrigPrecision() const { return keepOriginalPrecision; } + bool isBF16ConversionDisabled() const { + return disableBF16Conversion; + } protected: bool canFuseSimpleOperation(const NodePtr& node) const; @@ -721,6 +724,7 @@ class Node { MemoryFormatFilter memoryFormatFilter; bool enforceBF16evenForGraphTail = false; bool keepOriginalPrecision = false; + bool disableBF16Conversion = false; std::string originalLayers; // contains names of the original layers separated by comma diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp new file mode 100644 index 00000000000000..16b9b63fc09b52 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp @@ -0,0 +1,101 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "disable_bf16_comp_cumsum_sin_gen.hpp" + +#include +#include +#include +#include + +#include "openvino/cc/pass/itt.hpp" +#include "openvino/core/node.hpp" +#include "openvino/core/type.hpp" +#include "openvino/core/type/element_type.hpp" +#include "openvino/op/cum_sum.hpp" +#include "openvino/op/interpolate.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/sin.hpp" +#include "openvino/op/transpose.hpp" +#include "openvino/pass/pattern/matcher.hpp" +#include "openvino/pass/pattern/op/pattern.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" + +namespace ov::intel_cpu { + +namespace { + +bool is_l_sin_gen_node(const std::shared_ptr& node) { + if (!node) { + return false; + } + return node->get_friendly_name().find("l_sin_gen") != std::string::npos; +} + +void mark_fp32_chain(const std::vector>& nodes) { + for (const auto& node : nodes) { + if (node) { + ov::disable_conversion(node, ov::element::f32, ov::element::bf16); + } + } +} + +} // namespace + +DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { + MATCHER_SCOPE(DisableBF16CompCumSumSinGen); + using namespace ov::pass::pattern; + + auto transpose_pre_m = wrap_type({any_input(), any_input()}); + + auto interp_pre_m = + wrap_type({transpose_pre_m, any_input(), any_input()}); + + auto transpose1_m = wrap_type({interp_pre_m, any_input()}); + auto cumsum_m = wrap_type({transpose1_m, any_input()}); + auto mul1_m = wrap_type({cumsum_m, any_input()}); + auto transpose2_m = wrap_type({mul1_m, any_input()}); + auto mul2_m = wrap_type({transpose2_m, any_input()}); + + auto interp_down_m = + wrap_type({mul2_m, any_input(), any_input()}); + + auto transpose3_m = wrap_type({interp_down_m, any_input()}); + auto sin_m = wrap_type({transpose3_m}); + + ov::matcher_pass_callback callback = [=](Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + + auto sin_node = pattern_map.at(sin_m).get_node_shared_ptr(); + if (transformation_callback(sin_node)) { + return false; + } + + if (!is_l_sin_gen_node(sin_node)) { + return false; + } + + std::vector> to_mark{ + pattern_map.at(transpose_pre_m).get_node_shared_ptr(), + pattern_map.at(interp_pre_m).get_node_shared_ptr(), + pattern_map.at(transpose1_m).get_node_shared_ptr(), + pattern_map.at(cumsum_m).get_node_shared_ptr(), + pattern_map.at(mul1_m).get_node_shared_ptr(), + pattern_map.at(transpose2_m).get_node_shared_ptr(), + pattern_map.at(mul2_m).get_node_shared_ptr(), + pattern_map.at(interp_down_m).get_node_shared_ptr(), + pattern_map.at(transpose3_m).get_node_shared_ptr(), + }; + + mark_fp32_chain(to_mark); + + return true; + }; + + auto m = std::make_shared(sin_m, matcher_name); + this->register_matcher(m, callback); +} + +} // namespace ov::intel_cpu diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp new file mode 100644 index 00000000000000..31985b977b19f4 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp @@ -0,0 +1,38 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/matcher_pass.hpp" + +namespace ov::intel_cpu { + +/** + * @brief Keeps the F0 oscillator chain (StyleTTS2 / iSTFTNet's `l_sin_gen`) + * in fp32 when the CPU plugin infers in BF16. The chain integrates + * phase through CumSum and produces a periodic excitation via Sin; + * any BF16 rounding along the phase path drifts the accumulated + * phase and audibly distorts the vocoder output. + * + * The pass matches the fixed l_sin_gen core topology below and directly marks + * the matched core nodes with @ref ov::disable_conversion(node, f32, bf16). + * CPU's EnforceInferencePrecision then expands this into a continuous fp32 + * island (including the upstream phase-preparation ops) during precision + * enforcement. + * + * Transpose -> Interpolate -> Transpose -> CumSum + * -> Multiply -> Transpose -> Multiply + * -> Interpolate -> Transpose -> Sin + * + * The pass is intentionally scoped to nodes with `l_sin_gen` in friendly + * names to avoid matching unrelated structurally similar chains. + * + */ +class DisableBF16CompCumSumSinGen : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("DisableBF16CompCumSumSinGen"); + DisableBF16CompCumSumSinGen(); +}; + +} // namespace ov::intel_cpu diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index 3d0275a98c6f16..7cb29855b583f9 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -145,6 +145,7 @@ #include "transformations/low_precision/mark_dequantization_subgraph.hpp" // CPU specific transformations +#include "transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp" #include "transformations/cpu_opset/common/pass/insert_convert_after_extension.hpp" #include "transformations/cpu_opset/common/pass/ngram_fusion.hpp" #include "transformations/cpu_opset/common/pass/permute_slice_n_interpolation.hpp" @@ -594,6 +595,7 @@ void Transformations::PreLpt(const std::vector& defaultPrecis CPU_REGISTER_PASS_COMMON(manager, ov::pass::AUGRUCellFusion); CPU_REGISTER_PASS_COMMON(manager, SDPASubgraphFusion); CPU_REGISTER_PASS_COMMON(manager, ov::pass::GatedDeltaNetFusion); + CPU_REGISTER_PASS_COMMON(manager, ov::intel_cpu::DisableBF16CompCumSumSinGen); CPU_REGISTER_PASS_COMMON(manager, ov::pass::CommonOptimizations); CPU_REGISTER_PASS_COMMON(manager, ov::pass::KeepConstPrecision, decompression_precisions, false, true); CPU_SET_CALLBACK_COMMON( diff --git a/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp b/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp new file mode 100644 index 00000000000000..8332fb5a413e4a --- /dev/null +++ b/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp @@ -0,0 +1,203 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp" + +#include + +#include +#include +#include + +#include "openvino/core/model.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/cum_sum.hpp" +#include "openvino/op/interpolate.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/sin.hpp" +#include "openvino/op/transpose.hpp" +#include "openvino/pass/manager.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" + +using namespace testing; +using namespace ov::intel_cpu; + +namespace { + +const std::string l_sin_gen_prefix = "__module.decoder.generator.m_source.l_sin_gen/"; + +// Names used to look up matched nodes after the pass has run. +const std::string name_transpose_pre = "transpose_pre"; +const std::string name_interp_pre = "interp_pre"; +const std::string name_transpose_pre_cumsum = "transpose_pre_cumsum"; +const std::string name_cumsum = "cumsum"; +const std::string name_mul_after_cumsum = "mul_after_cumsum"; +const std::string name_transpose_after_mul = "transpose_after_mul"; +const std::string name_scale_mul = "scale_mul"; +const std::string name_interp_after = "interp_after"; +const std::string name_transpose_after_interp = "transpose_after_interp"; +const std::string name_sin = l_sin_gen_prefix + "aten::sin/Sin"; + +ov::op::util::InterpolateBase::InterpolateAttrs make_interp_attrs() { + ov::op::util::InterpolateBase::InterpolateAttrs attrs; + attrs.mode = ov::op::util::InterpolateBase::InterpolateMode::LINEAR; + attrs.shape_calculation_mode = ov::op::util::InterpolateBase::ShapeCalcMode::SCALES; + attrs.nearest_mode = ov::op::util::InterpolateBase::NearestMode::ROUND_PREFER_FLOOR; + return attrs; +} + +// Transpose -> Interpolate -> Transpose -> CumSum +// -> Multiply -> Transpose -> Multiply -> Interpolate -> Transpose -> Sin. +std::shared_ptr create_full_chain_model() { + auto input = std::make_shared(ov::element::f32, ov::PartialShape{1, 32}); + + auto order_pre = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_pre = std::make_shared(input, order_pre); + transpose_pre->set_friendly_name(name_transpose_pre); + + auto sizes_pre = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {64L, 1L}); + auto scales_pre = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, {2.0f, 1.0f}); + auto interp_pre = + std::make_shared(transpose_pre, sizes_pre, scales_pre, make_interp_attrs()); + interp_pre->set_friendly_name(name_interp_pre); + + auto order_pre_cumsum = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_pre_cumsum = std::make_shared(interp_pre, order_pre_cumsum); + transpose_pre_cumsum->set_friendly_name(name_transpose_pre_cumsum); + + auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}); + auto cumsum = std::make_shared(transpose_pre_cumsum, axis); + cumsum->set_friendly_name(name_cumsum); + + auto mul_after_cumsum_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1, 64}, {6.2832f}); + auto mul_after_cumsum = std::make_shared(cumsum, mul_after_cumsum_const); + mul_after_cumsum->set_friendly_name(name_mul_after_cumsum); + + auto order_after_mul = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_after_mul = std::make_shared(mul_after_cumsum, order_after_mul); + transpose_after_mul->set_friendly_name(name_transpose_after_mul); + + auto scale_mul_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{64, 1}, {1.0f}); + auto scale_mul = std::make_shared(transpose_after_mul, scale_mul_const); + scale_mul->set_friendly_name(name_scale_mul); + + auto sizes_after = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {128L, 1L}); + auto scales_after = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, {2.0f, 1.0f}); + auto interp_after = + std::make_shared(scale_mul, sizes_after, scales_after, make_interp_attrs()); + interp_after->set_friendly_name(name_interp_after); + + auto order_after_interp = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_after_interp = std::make_shared(interp_after, order_after_interp); + transpose_after_interp->set_friendly_name(name_transpose_after_interp); + + auto sin = std::make_shared(transpose_after_interp); + sin->set_friendly_name(name_sin); + + return std::make_shared(ov::OutputVector{sin}, ov::ParameterVector{input}); +} + +// Same downstream chain but missing the second Multiply between the two +// Transposes — must not match. +std::shared_ptr create_model_missing_scale_mul() { + auto input = std::make_shared(ov::element::f32, ov::PartialShape{1, 32}); + + auto order_pre = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_pre = std::make_shared(input, order_pre); + transpose_pre->set_friendly_name(name_transpose_pre); + + auto sizes_pre = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {64L, 1L}); + auto scales_pre = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, {2.0f, 1.0f}); + auto interp_pre = + std::make_shared(transpose_pre, sizes_pre, scales_pre, make_interp_attrs()); + interp_pre->set_friendly_name(name_interp_pre); + + auto order_pre_cumsum = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_pre_cumsum = std::make_shared(interp_pre, order_pre_cumsum); + transpose_pre_cumsum->set_friendly_name(name_transpose_pre_cumsum); + + auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}); + auto cumsum = std::make_shared(transpose_pre_cumsum, axis); + cumsum->set_friendly_name(name_cumsum); + + auto mul_after_cumsum_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1, 64}, {6.2832f}); + auto mul_after_cumsum = std::make_shared(cumsum, mul_after_cumsum_const); + mul_after_cumsum->set_friendly_name(name_mul_after_cumsum); + + auto order_after_mul = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_after_mul = std::make_shared(mul_after_cumsum, order_after_mul); + transpose_after_mul->set_friendly_name(name_transpose_after_mul); + + // Missing scale Multiply — Interpolate is fed directly by the Transpose. + auto sizes_after = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {64L, 1L}); + auto scales_after = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, {2.0f, 1.0f}); + auto interp_after = + std::make_shared(transpose_after_mul, sizes_after, scales_after, make_interp_attrs()); + interp_after->set_friendly_name(name_interp_after); + + auto order_after_interp = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_after_interp = std::make_shared(interp_after, order_after_interp); + transpose_after_interp->set_friendly_name(name_transpose_after_interp); + + auto sin = std::make_shared(transpose_after_interp); + sin->set_friendly_name(name_sin); + + return std::make_shared(ov::OutputVector{sin}, ov::ParameterVector{input}); +} + +void run_test(const std::shared_ptr& model, + const std::unordered_map& expected_bf16_disabled_status) { + ov::pass::Manager manager; + manager.register_pass(); + manager.run_passes(model); + + for (const auto& op : model->get_ops()) { + auto it = expected_bf16_disabled_status.find(op->get_friendly_name()); + if (it == expected_bf16_disabled_status.end()) { + continue; + } + if (it->second) { + ASSERT_TRUE(ov::is_conversion_disabled(op, ov::element::f32, ov::element::bf16)) + << "BF16 conversion is not disabled for node: " << op->get_friendly_name(); + } else { + ASSERT_FALSE(ov::is_conversion_disabled(op, ov::element::f32, ov::element::bf16)) + << "BF16 conversion is unexpectedly disabled for node: " << op->get_friendly_name(); + } + } +} + +} // namespace + +TEST(TransformationTests, DisableBF16CompCumSumSinGen_Positive) { + auto model = create_full_chain_model(); + // Core matched nodes are marked as disabled for BF16 conversion. + std::unordered_map expected_status = { + {name_transpose_pre, true}, + {name_interp_pre, true}, + {name_transpose_pre_cumsum, true}, + {name_cumsum, true}, + {name_mul_after_cumsum, true}, + {name_transpose_after_mul, true}, + {name_scale_mul, true}, + {name_interp_after, true}, + {name_transpose_after_interp, true}, + }; + run_test(model, expected_status); +} + +TEST(TransformationTests, DisableBF16CompCumSumSinGen_MissingScaleMultiply_NoOp) { + auto model = create_model_missing_scale_mul(); + std::unordered_map expected_status = { + {name_transpose_pre, false}, + {name_interp_pre, false}, + {name_transpose_pre_cumsum, false}, + {name_cumsum, false}, + {name_mul_after_cumsum, false}, + {name_transpose_after_mul, false}, + {name_interp_after, false}, + {name_transpose_after_interp, false}, + }; + run_test(model, expected_status); +} From 7eb47e547aef16a81f20d8fa60308a2473e40b79 Mon Sep 17 00:00:00 2001 From: mangguo Date: Fri, 7 Aug 2026 10:23:01 +0800 Subject: [PATCH 2/8] Fix review comments. Add test case. --- src/plugins/intel_cpu/src/graph.cpp | 5 +- .../pass/disable_bf16_comp_cumsum_sin_gen.cpp | 25 ++++-- .../enforce_inference_precision_bf16_test.cpp | 88 +++++++++++++++++++ .../disable_bf16_comp_cumsum_sin_gen_test.cpp | 2 + 4 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 src/plugins/intel_cpu/tests/unit/graph/enforce_inference_precision_bf16_test.cpp diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index f0b336d46f5ccd..c3a27483f3d4f0 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -2097,7 +2097,10 @@ void Graph::EnforceInferencePrecision() const { if (!node->isBF16ConversionDisabled()) { continue; } - nodesToSkip.insert(node); + const auto inserted = nodesToSkip.insert(node); + if (!inserted.second) { + continue; + } backwardSkipSearch(node, nodesToSkip); forwardSkipSearch(node, nodesToSkip); } diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp index 16b9b63fc09b52..a18be002b3cd85 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp @@ -4,21 +4,22 @@ #include "disable_bf16_comp_cumsum_sin_gen.hpp" -#include #include #include #include #include "openvino/cc/pass/itt.hpp" #include "openvino/core/node.hpp" -#include "openvino/core/type.hpp" #include "openvino/core/type/element_type.hpp" #include "openvino/op/cum_sum.hpp" #include "openvino/op/interpolate.hpp" #include "openvino/op/multiply.hpp" #include "openvino/op/sin.hpp" +#include "openvino/pass/matcher_pass.hpp" #include "openvino/op/transpose.hpp" #include "openvino/pass/pattern/matcher.hpp" +#include "openvino/pass/pattern/op/label.hpp" +#include "openvino/pass/pattern/op/or.hpp" #include "openvino/pass/pattern/op/pattern.hpp" #include "openvino/pass/pattern/op/wrap_type.hpp" #include "transformations/rt_info/disable_precision_conversion.hpp" @@ -50,8 +51,11 @@ DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { auto transpose_pre_m = wrap_type({any_input(), any_input()}); - auto interp_pre_m = + auto interp_pre_3_m = wrap_type({transpose_pre_m, any_input(), any_input()}); + auto interp_pre_4_m = wrap_type( + {transpose_pre_m, any_input(), any_input(), any_input()}); + auto interp_pre_m = std::make_shared(OutputVector{interp_pre_3_m, interp_pre_4_m}); auto transpose1_m = wrap_type({interp_pre_m, any_input()}); auto cumsum_m = wrap_type({transpose1_m, any_input()}); @@ -59,8 +63,11 @@ DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { auto transpose2_m = wrap_type({mul1_m, any_input()}); auto mul2_m = wrap_type({transpose2_m, any_input()}); - auto interp_down_m = + auto interp_down_3_m = wrap_type({mul2_m, any_input(), any_input()}); + auto interp_down_4_m = + wrap_type({mul2_m, any_input(), any_input(), any_input()}); + auto interp_down_m = std::make_shared(OutputVector{interp_down_3_m, interp_down_4_m}); auto transpose3_m = wrap_type({interp_down_m, any_input()}); auto sin_m = wrap_type({transpose3_m}); @@ -79,16 +86,22 @@ DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { std::vector> to_mark{ pattern_map.at(transpose_pre_m).get_node_shared_ptr(), - pattern_map.at(interp_pre_m).get_node_shared_ptr(), pattern_map.at(transpose1_m).get_node_shared_ptr(), pattern_map.at(cumsum_m).get_node_shared_ptr(), pattern_map.at(mul1_m).get_node_shared_ptr(), pattern_map.at(transpose2_m).get_node_shared_ptr(), pattern_map.at(mul2_m).get_node_shared_ptr(), - pattern_map.at(interp_down_m).get_node_shared_ptr(), pattern_map.at(transpose3_m).get_node_shared_ptr(), + sin_node, }; + for (const auto& key : {interp_pre_3_m, interp_pre_4_m, interp_down_3_m, interp_down_4_m}) { + auto it = pattern_map.find(key); + if (it != pattern_map.end()) { + to_mark.push_back(it->second.get_node_shared_ptr()); + } + } + mark_fp32_chain(to_mark); return true; diff --git a/src/plugins/intel_cpu/tests/unit/graph/enforce_inference_precision_bf16_test.cpp b/src/plugins/intel_cpu/tests/unit/graph/enforce_inference_precision_bf16_test.cpp new file mode 100644 index 00000000000000..2905d6e2fe1b21 --- /dev/null +++ b/src/plugins/intel_cpu/tests/unit/graph/enforce_inference_precision_bf16_test.cpp @@ -0,0 +1,88 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include + +#include "graph.h" +#include "openvino/core/model.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/convert.hpp" +#include "openvino/op/cum_sum.hpp" +#include "openvino/op/matmul.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/result.hpp" +#include "openvino/op/softmax.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" + +using namespace ov::intel_cpu; + +namespace { + +const std::string matmul_before_name = "matmul_before"; +const std::string cumsum_name = "cumsum"; +const std::string softmax_name = "softmax"; +const std::string matmul_after_name = "matmul_after"; + +NodePtr find_graph_node_by_name(const Graph& graph, const std::string& name) { + for (const auto& node : graph.GetNodes()) { + if (node->getName() == name || node->getOriginalLayers().find(name) != std::string::npos) { + return node; + } + } + return nullptr; +} + +std::shared_ptr create_model_with_tagged_node() { + const auto shape = ov::Shape{1, 32}; + const auto weights_shape = ov::Shape{32, 32}; + auto input = std::make_shared(ov::element::f32, shape); + auto matmul_before = + std::make_shared(input, + ov::op::v0::Constant::create(ov::element::f32, weights_shape, {1.0f})); + matmul_before->set_friendly_name(matmul_before_name); + + auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); + auto cumsum = std::make_shared(matmul_before, axis); + cumsum->set_friendly_name(cumsum_name); + ov::disable_conversion(cumsum, ov::element::f32, ov::element::bf16); + + auto island_softmax = std::make_shared(cumsum, 1); + island_softmax->set_friendly_name(softmax_name); + + auto matmul_after = + std::make_shared(island_softmax, + ov::op::v0::Constant::create(ov::element::f32, weights_shape, {1.0f})); + matmul_after->set_friendly_name(matmul_after_name); + auto output = std::make_shared(matmul_after, ov::element::bf16); + + return std::make_shared(ov::ResultVector{std::make_shared(output)}, + ov::ParameterVector{input}); +} + +void expect_node_precision(const NodePtr& node, const ov::element::Type& precision) { + ASSERT_NE(node, nullptr); + EXPECT_EQ(node->getOriginalInputPrecisionAtPort(0), precision) << node->getName(); + EXPECT_EQ(node->getOriginalOutputPrecisionAtPort(0), precision) << node->getName(); +} + +} // namespace + +TEST(EnforceInferencePrecisionBF16Test, KeepsFp32IslandBetweenMandatoryBF16Nodes) { + Config config; + config.inferencePrecision = ov::element::bf16; + config.inferencePrecisionSetExplicitly = true; + + auto context = std::make_shared(config, nullptr, false); + Graph graph; + graph.Init(create_model_with_tagged_node(), context); + + expect_node_precision(find_graph_node_by_name(graph, matmul_before_name), ov::element::bf16); + expect_node_precision(find_graph_node_by_name(graph, cumsum_name), ov::element::f32); + expect_node_precision(find_graph_node_by_name(graph, softmax_name), ov::element::f32); + expect_node_precision(find_graph_node_by_name(graph, matmul_after_name), ov::element::bf16); +} diff --git a/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp b/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp index 8332fb5a413e4a..d294ab669c8950 100644 --- a/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp @@ -183,6 +183,7 @@ TEST(TransformationTests, DisableBF16CompCumSumSinGen_Positive) { {name_scale_mul, true}, {name_interp_after, true}, {name_transpose_after_interp, true}, + {name_sin, true}, }; run_test(model, expected_status); } @@ -198,6 +199,7 @@ TEST(TransformationTests, DisableBF16CompCumSumSinGen_MissingScaleMultiply_NoOp) {name_transpose_after_mul, false}, {name_interp_after, false}, {name_transpose_after_interp, false}, + {name_sin, false}, }; run_test(model, expected_status); } From df1e2d8d99e4978d15e5cfbed22ee72e51db9f97 Mon Sep 17 00:00:00 2001 From: mangguo Date: Fri, 7 Aug 2026 10:57:43 +0800 Subject: [PATCH 3/8] Fix clang-format error --- .../cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp index a18be002b3cd85..0206f2e266ff49 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp @@ -15,8 +15,8 @@ #include "openvino/op/interpolate.hpp" #include "openvino/op/multiply.hpp" #include "openvino/op/sin.hpp" -#include "openvino/pass/matcher_pass.hpp" #include "openvino/op/transpose.hpp" +#include "openvino/pass/matcher_pass.hpp" #include "openvino/pass/pattern/matcher.hpp" #include "openvino/pass/pattern/op/label.hpp" #include "openvino/pass/pattern/op/or.hpp" From 20307bd4ed8d67bf889424daa057b073a7d40390 Mon Sep 17 00:00:00 2001 From: mangguo Date: Fri, 7 Aug 2026 15:50:28 +0800 Subject: [PATCH 4/8] Add Interpolate_v0 support. Apply review comments. --- .../pass/disable_bf16_comp_cumsum_sin_gen.cpp | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp index 0206f2e266ff49..317ed551f86f64 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp @@ -35,14 +35,6 @@ bool is_l_sin_gen_node(const std::shared_ptr& node) { return node->get_friendly_name().find("l_sin_gen") != std::string::npos; } -void mark_fp32_chain(const std::vector>& nodes) { - for (const auto& node : nodes) { - if (node) { - ov::disable_conversion(node, ov::element::f32, ov::element::bf16); - } - } -} - } // namespace DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { @@ -51,11 +43,19 @@ DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { auto transpose_pre_m = wrap_type({any_input(), any_input()}); - auto interp_pre_3_m = - wrap_type({transpose_pre_m, any_input(), any_input()}); - auto interp_pre_4_m = wrap_type( - {transpose_pre_m, any_input(), any_input(), any_input()}); - auto interp_pre_m = std::make_shared(OutputVector{interp_pre_3_m, interp_pre_4_m}); + auto interp_pre_v0_m = wrap_type({transpose_pre_m, any_input()}); + auto interp_pre_v4_m = wrap_type({transpose_pre_m, any_input(), any_input()}); + auto interp_pre_v4_with_axes_m = + wrap_type({transpose_pre_m, any_input(), any_input(), any_input()}); + auto interp_pre_v11_m = wrap_type({transpose_pre_m, any_input()}); + auto interp_pre_v11_with_axes_m = + wrap_type({transpose_pre_m, any_input(), any_input()}); + auto interp_pre_m = std::make_shared( + OutputVector{interp_pre_v0_m, + interp_pre_v4_m, + interp_pre_v4_with_axes_m, + interp_pre_v11_m, + interp_pre_v11_with_axes_m}); auto transpose1_m = wrap_type({interp_pre_m, any_input()}); auto cumsum_m = wrap_type({transpose1_m, any_input()}); @@ -63,11 +63,17 @@ DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { auto transpose2_m = wrap_type({mul1_m, any_input()}); auto mul2_m = wrap_type({transpose2_m, any_input()}); - auto interp_down_3_m = - wrap_type({mul2_m, any_input(), any_input()}); - auto interp_down_4_m = - wrap_type({mul2_m, any_input(), any_input(), any_input()}); - auto interp_down_m = std::make_shared(OutputVector{interp_down_3_m, interp_down_4_m}); + auto interp_down_v0_m = wrap_type({mul2_m, any_input()}); + auto interp_down_v4_m = wrap_type({mul2_m, any_input(), any_input()}); + auto interp_down_v4_with_axes_m = wrap_type({mul2_m, any_input(), any_input(), any_input()}); + auto interp_down_v11_m = wrap_type({mul2_m, any_input()}); + auto interp_down_v11_with_axes_m = wrap_type({mul2_m, any_input(), any_input()}); + auto interp_down_m = std::make_shared( + OutputVector{interp_down_v0_m, + interp_down_v4_m, + interp_down_v4_with_axes_m, + interp_down_v11_m, + interp_down_v11_with_axes_m}); auto transpose3_m = wrap_type({interp_down_m, any_input()}); auto sin_m = wrap_type({transpose3_m}); @@ -86,24 +92,23 @@ DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { std::vector> to_mark{ pattern_map.at(transpose_pre_m).get_node_shared_ptr(), + pattern_map.at(interp_pre_m).get_node_shared_ptr(), pattern_map.at(transpose1_m).get_node_shared_ptr(), pattern_map.at(cumsum_m).get_node_shared_ptr(), pattern_map.at(mul1_m).get_node_shared_ptr(), pattern_map.at(transpose2_m).get_node_shared_ptr(), pattern_map.at(mul2_m).get_node_shared_ptr(), + pattern_map.at(interp_down_m).get_node_shared_ptr(), pattern_map.at(transpose3_m).get_node_shared_ptr(), sin_node, }; - for (const auto& key : {interp_pre_3_m, interp_pre_4_m, interp_down_3_m, interp_down_4_m}) { - auto it = pattern_map.find(key); - if (it != pattern_map.end()) { - to_mark.push_back(it->second.get_node_shared_ptr()); + for (const auto& node : to_mark) { + if (node) { + ov::disable_conversion(node, ov::element::f32, ov::element::bf16); } } - mark_fp32_chain(to_mark); - return true; }; From 3dcd3450b0e65f5e3212165aa982334fb4678581 Mon Sep 17 00:00:00 2001 From: mangguo Date: Fri, 7 Aug 2026 16:25:05 +0800 Subject: [PATCH 5/8] Remove the node friendly name check. --- .../pass/disable_bf16_comp_cumsum_sin_gen.cpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp index 317ed551f86f64..e0eb47256edc9d 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp @@ -5,7 +5,6 @@ #include "disable_bf16_comp_cumsum_sin_gen.hpp" #include -#include #include #include "openvino/cc/pass/itt.hpp" @@ -26,17 +25,6 @@ namespace ov::intel_cpu { -namespace { - -bool is_l_sin_gen_node(const std::shared_ptr& node) { - if (!node) { - return false; - } - return node->get_friendly_name().find("l_sin_gen") != std::string::npos; -} - -} // namespace - DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { MATCHER_SCOPE(DisableBF16CompCumSumSinGen); using namespace ov::pass::pattern; @@ -86,10 +74,6 @@ DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { return false; } - if (!is_l_sin_gen_node(sin_node)) { - return false; - } - std::vector> to_mark{ pattern_map.at(transpose_pre_m).get_node_shared_ptr(), pattern_map.at(interp_pre_m).get_node_shared_ptr(), From 544d87e03fd81905d738a887255c330d57b3d708 Mon Sep 17 00:00:00 2001 From: mangguo Date: Mon, 10 Aug 2026 13:44:04 +0800 Subject: [PATCH 6/8] Apply review comments --- .../pass/disable_bf16_comp_cumsum_sin_gen.cpp | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp index e0eb47256edc9d..c8db13c893a00e 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp @@ -28,22 +28,19 @@ namespace ov::intel_cpu { DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { MATCHER_SCOPE(DisableBF16CompCumSumSinGen); using namespace ov::pass::pattern; + using ov::pass::operator|; + + auto interpolate_variations = [](const ov::Output& input) { + auto interp_v0_m = wrap_type({input, any_input()}); + auto interp_v4_m = wrap_type({input, any_input(), any_input()}); + auto interp_v4_with_axes_m = wrap_type({input, any_input(), any_input(), any_input()}); + auto interp_v11_m = wrap_type({input, any_input()}); + auto interp_v11_with_axes_m = wrap_type({input, any_input(), any_input()}); + return interp_v0_m | interp_v4_m | interp_v4_with_axes_m | interp_v11_m | interp_v11_with_axes_m; + }; auto transpose_pre_m = wrap_type({any_input(), any_input()}); - - auto interp_pre_v0_m = wrap_type({transpose_pre_m, any_input()}); - auto interp_pre_v4_m = wrap_type({transpose_pre_m, any_input(), any_input()}); - auto interp_pre_v4_with_axes_m = - wrap_type({transpose_pre_m, any_input(), any_input(), any_input()}); - auto interp_pre_v11_m = wrap_type({transpose_pre_m, any_input()}); - auto interp_pre_v11_with_axes_m = - wrap_type({transpose_pre_m, any_input(), any_input()}); - auto interp_pre_m = std::make_shared( - OutputVector{interp_pre_v0_m, - interp_pre_v4_m, - interp_pre_v4_with_axes_m, - interp_pre_v11_m, - interp_pre_v11_with_axes_m}); + auto interp_pre_m = interpolate_variations(transpose_pre_m); auto transpose1_m = wrap_type({interp_pre_m, any_input()}); auto cumsum_m = wrap_type({transpose1_m, any_input()}); @@ -51,17 +48,7 @@ DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { auto transpose2_m = wrap_type({mul1_m, any_input()}); auto mul2_m = wrap_type({transpose2_m, any_input()}); - auto interp_down_v0_m = wrap_type({mul2_m, any_input()}); - auto interp_down_v4_m = wrap_type({mul2_m, any_input(), any_input()}); - auto interp_down_v4_with_axes_m = wrap_type({mul2_m, any_input(), any_input(), any_input()}); - auto interp_down_v11_m = wrap_type({mul2_m, any_input()}); - auto interp_down_v11_with_axes_m = wrap_type({mul2_m, any_input(), any_input()}); - auto interp_down_m = std::make_shared( - OutputVector{interp_down_v0_m, - interp_down_v4_m, - interp_down_v4_with_axes_m, - interp_down_v11_m, - interp_down_v11_with_axes_m}); + auto interp_down_m = interpolate_variations(mul2_m); auto transpose3_m = wrap_type({interp_down_m, any_input()}); auto sin_m = wrap_type({transpose3_m}); From a18c78aaa4a8fc043c286b3dcb7d86f099f095bb Mon Sep 17 00:00:00 2001 From: mangguo Date: Mon, 10 Aug 2026 14:15:31 +0800 Subject: [PATCH 7/8] Fix Clang format error --- .../cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp index c8db13c893a00e..28add079faa116 100644 --- a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp @@ -9,6 +9,7 @@ #include "openvino/cc/pass/itt.hpp" #include "openvino/core/node.hpp" +#include "openvino/core/node_output.hpp" #include "openvino/core/type/element_type.hpp" #include "openvino/op/cum_sum.hpp" #include "openvino/op/interpolate.hpp" From 3b84acb0a8db580c8c98e3e3439926b0d109a539 Mon Sep 17 00:00:00 2001 From: mangguo Date: Fri, 14 Aug 2026 09:45:35 +0800 Subject: [PATCH 8/8] Change sin node name in test case --- .../transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp b/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp index d294ab669c8950..6f1a7030215ae9 100644 --- a/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp +++ b/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp @@ -26,8 +26,6 @@ using namespace ov::intel_cpu; namespace { -const std::string l_sin_gen_prefix = "__module.decoder.generator.m_source.l_sin_gen/"; - // Names used to look up matched nodes after the pass has run. const std::string name_transpose_pre = "transpose_pre"; const std::string name_interp_pre = "interp_pre"; @@ -38,7 +36,7 @@ const std::string name_transpose_after_mul = "transpose_after_mul"; const std::string name_scale_mul = "scale_mul"; const std::string name_interp_after = "interp_after"; const std::string name_transpose_after_interp = "transpose_after_interp"; -const std::string name_sin = l_sin_gen_prefix + "aten::sin/Sin"; +const std::string name_sin = "sin"; ov::op::util::InterpolateBase::InterpolateAttrs make_interp_attrs() { ov::op::util::InterpolateBase::InterpolateAttrs attrs;