diff --git a/src/plugins/intel_cpu/src/graph.cpp b/src/plugins/intel_cpu/src/graph.cpp index 5bb6a7ab4401f7..c3a27483f3d4f0 100644 --- a/src/plugins/intel_cpu/src/graph.cpp +++ b/src/plugins/intel_cpu/src/graph.cpp @@ -2092,6 +2092,19 @@ void Graph::EnforceInferencePrecision() const { // Pattern-based node skipping for BF16 precision enforcement if (inferPrec == ov::element::bf16) { + // Preserve a continuous f32 region around model-declared precision-sensitive operations. + for (const auto& node : graphNodes) { + if (!node->isBF16ConversionDisabled()) { + continue; + } + const auto inserted = nodesToSkip.insert(node); + if (!inserted.second) { + continue; + } + backwardSkipSearch(node, nodesToSkip); + forwardSkipSearch(node, nodesToSkip); + } + for (const auto& node : graphNodes) { // Pattern 1: MatMul with Convert from integer to floating point on any input. This basically means that // converting such an integer input to bf16 leads to loosing accuracy, as bf16 can only exactly represent diff --git a/src/plugins/intel_cpu/src/node.cpp b/src/plugins/intel_cpu/src/node.cpp index 15be8a84cfb992..39651590d5da0f 100644 --- a/src/plugins/intel_cpu/src/node.cpp +++ b/src/plugins/intel_cpu/src/node.cpp @@ -190,6 +190,7 @@ Node::Node(const std::shared_ptr& op, GraphContext::CPtr ctx, const Sh if (is_conversion_disabled(op, element::f16)) { keepOriginalPrecision = true; } + disableBF16Conversion = is_conversion_disabled(op, element::f32, element::bf16); } Node::Node(const std::string& type, diff --git a/src/plugins/intel_cpu/src/node.h b/src/plugins/intel_cpu/src/node.h index b867ea31a8e974..f2ef5cc70d0482 100644 --- a/src/plugins/intel_cpu/src/node.h +++ b/src/plugins/intel_cpu/src/node.h @@ -690,6 +690,9 @@ class Node { bool keepOrigPrecision() const { return keepOriginalPrecision; } + bool isBF16ConversionDisabled() const { + return disableBF16Conversion; + } protected: bool canFuseSimpleOperation(const NodePtr& node) const; @@ -721,6 +724,7 @@ class Node { MemoryFormatFilter memoryFormatFilter; bool enforceBF16evenForGraphTail = false; bool keepOriginalPrecision = false; + bool disableBF16Conversion = false; std::string originalLayers; // contains names of the original layers separated by comma diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp new file mode 100644 index 00000000000000..28add079faa116 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.cpp @@ -0,0 +1,91 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "disable_bf16_comp_cumsum_sin_gen.hpp" + +#include +#include + +#include "openvino/cc/pass/itt.hpp" +#include "openvino/core/node.hpp" +#include "openvino/core/node_output.hpp" +#include "openvino/core/type/element_type.hpp" +#include "openvino/op/cum_sum.hpp" +#include "openvino/op/interpolate.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/sin.hpp" +#include "openvino/op/transpose.hpp" +#include "openvino/pass/matcher_pass.hpp" +#include "openvino/pass/pattern/matcher.hpp" +#include "openvino/pass/pattern/op/label.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/pattern/op/pattern.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" + +namespace ov::intel_cpu { + +DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { + MATCHER_SCOPE(DisableBF16CompCumSumSinGen); + using namespace ov::pass::pattern; + using ov::pass::operator|; + + auto interpolate_variations = [](const ov::Output& input) { + auto interp_v0_m = wrap_type({input, any_input()}); + auto interp_v4_m = wrap_type({input, any_input(), any_input()}); + auto interp_v4_with_axes_m = wrap_type({input, any_input(), any_input(), any_input()}); + auto interp_v11_m = wrap_type({input, any_input()}); + auto interp_v11_with_axes_m = wrap_type({input, any_input(), any_input()}); + return interp_v0_m | interp_v4_m | interp_v4_with_axes_m | interp_v11_m | interp_v11_with_axes_m; + }; + + auto transpose_pre_m = wrap_type({any_input(), any_input()}); + auto interp_pre_m = interpolate_variations(transpose_pre_m); + + auto transpose1_m = wrap_type({interp_pre_m, any_input()}); + auto cumsum_m = wrap_type({transpose1_m, any_input()}); + auto mul1_m = wrap_type({cumsum_m, any_input()}); + auto transpose2_m = wrap_type({mul1_m, any_input()}); + auto mul2_m = wrap_type({transpose2_m, any_input()}); + + auto interp_down_m = interpolate_variations(mul2_m); + + auto transpose3_m = wrap_type({interp_down_m, any_input()}); + auto sin_m = wrap_type({transpose3_m}); + + ov::matcher_pass_callback callback = [=](Matcher& m) { + const auto& pattern_map = m.get_pattern_value_map(); + + auto sin_node = pattern_map.at(sin_m).get_node_shared_ptr(); + if (transformation_callback(sin_node)) { + return false; + } + + std::vector> to_mark{ + pattern_map.at(transpose_pre_m).get_node_shared_ptr(), + pattern_map.at(interp_pre_m).get_node_shared_ptr(), + pattern_map.at(transpose1_m).get_node_shared_ptr(), + pattern_map.at(cumsum_m).get_node_shared_ptr(), + pattern_map.at(mul1_m).get_node_shared_ptr(), + pattern_map.at(transpose2_m).get_node_shared_ptr(), + pattern_map.at(mul2_m).get_node_shared_ptr(), + pattern_map.at(interp_down_m).get_node_shared_ptr(), + pattern_map.at(transpose3_m).get_node_shared_ptr(), + sin_node, + }; + + for (const auto& node : to_mark) { + if (node) { + ov::disable_conversion(node, ov::element::f32, ov::element::bf16); + } + } + + return true; + }; + + auto m = std::make_shared(sin_m, matcher_name); + this->register_matcher(m, callback); +} + +} // namespace ov::intel_cpu diff --git a/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp new file mode 100644 index 00000000000000..31985b977b19f4 --- /dev/null +++ b/src/plugins/intel_cpu/src/transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp @@ -0,0 +1,38 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "openvino/pass/matcher_pass.hpp" + +namespace ov::intel_cpu { + +/** + * @brief Keeps the F0 oscillator chain (StyleTTS2 / iSTFTNet's `l_sin_gen`) + * in fp32 when the CPU plugin infers in BF16. The chain integrates + * phase through CumSum and produces a periodic excitation via Sin; + * any BF16 rounding along the phase path drifts the accumulated + * phase and audibly distorts the vocoder output. + * + * The pass matches the fixed l_sin_gen core topology below and directly marks + * the matched core nodes with @ref ov::disable_conversion(node, f32, bf16). + * CPU's EnforceInferencePrecision then expands this into a continuous fp32 + * island (including the upstream phase-preparation ops) during precision + * enforcement. + * + * Transpose -> Interpolate -> Transpose -> CumSum + * -> Multiply -> Transpose -> Multiply + * -> Interpolate -> Transpose -> Sin + * + * The pass is intentionally scoped to nodes with `l_sin_gen` in friendly + * names to avoid matching unrelated structurally similar chains. + * + */ +class DisableBF16CompCumSumSinGen : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("DisableBF16CompCumSumSinGen"); + DisableBF16CompCumSumSinGen(); +}; + +} // namespace ov::intel_cpu diff --git a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp index 3d0275a98c6f16..7cb29855b583f9 100644 --- a/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp +++ b/src/plugins/intel_cpu/src/transformations/transformation_pipeline.cpp @@ -145,6 +145,7 @@ #include "transformations/low_precision/mark_dequantization_subgraph.hpp" // CPU specific transformations +#include "transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp" #include "transformations/cpu_opset/common/pass/insert_convert_after_extension.hpp" #include "transformations/cpu_opset/common/pass/ngram_fusion.hpp" #include "transformations/cpu_opset/common/pass/permute_slice_n_interpolation.hpp" @@ -594,6 +595,7 @@ void Transformations::PreLpt(const std::vector& defaultPrecis CPU_REGISTER_PASS_COMMON(manager, ov::pass::AUGRUCellFusion); CPU_REGISTER_PASS_COMMON(manager, SDPASubgraphFusion); CPU_REGISTER_PASS_COMMON(manager, ov::pass::GatedDeltaNetFusion); + CPU_REGISTER_PASS_COMMON(manager, ov::intel_cpu::DisableBF16CompCumSumSinGen); CPU_REGISTER_PASS_COMMON(manager, ov::pass::CommonOptimizations); CPU_REGISTER_PASS_COMMON(manager, ov::pass::KeepConstPrecision, decompression_precisions, false, true); CPU_SET_CALLBACK_COMMON( diff --git a/src/plugins/intel_cpu/tests/unit/graph/enforce_inference_precision_bf16_test.cpp b/src/plugins/intel_cpu/tests/unit/graph/enforce_inference_precision_bf16_test.cpp new file mode 100644 index 00000000000000..2905d6e2fe1b21 --- /dev/null +++ b/src/plugins/intel_cpu/tests/unit/graph/enforce_inference_precision_bf16_test.cpp @@ -0,0 +1,88 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include +#include +#include + +#include "graph.h" +#include "openvino/core/model.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/convert.hpp" +#include "openvino/op/cum_sum.hpp" +#include "openvino/op/matmul.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/result.hpp" +#include "openvino/op/softmax.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" + +using namespace ov::intel_cpu; + +namespace { + +const std::string matmul_before_name = "matmul_before"; +const std::string cumsum_name = "cumsum"; +const std::string softmax_name = "softmax"; +const std::string matmul_after_name = "matmul_after"; + +NodePtr find_graph_node_by_name(const Graph& graph, const std::string& name) { + for (const auto& node : graph.GetNodes()) { + if (node->getName() == name || node->getOriginalLayers().find(name) != std::string::npos) { + return node; + } + } + return nullptr; +} + +std::shared_ptr create_model_with_tagged_node() { + const auto shape = ov::Shape{1, 32}; + const auto weights_shape = ov::Shape{32, 32}; + auto input = std::make_shared(ov::element::f32, shape); + auto matmul_before = + std::make_shared(input, + ov::op::v0::Constant::create(ov::element::f32, weights_shape, {1.0f})); + matmul_before->set_friendly_name(matmul_before_name); + + auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); + auto cumsum = std::make_shared(matmul_before, axis); + cumsum->set_friendly_name(cumsum_name); + ov::disable_conversion(cumsum, ov::element::f32, ov::element::bf16); + + auto island_softmax = std::make_shared(cumsum, 1); + island_softmax->set_friendly_name(softmax_name); + + auto matmul_after = + std::make_shared(island_softmax, + ov::op::v0::Constant::create(ov::element::f32, weights_shape, {1.0f})); + matmul_after->set_friendly_name(matmul_after_name); + auto output = std::make_shared(matmul_after, ov::element::bf16); + + return std::make_shared(ov::ResultVector{std::make_shared(output)}, + ov::ParameterVector{input}); +} + +void expect_node_precision(const NodePtr& node, const ov::element::Type& precision) { + ASSERT_NE(node, nullptr); + EXPECT_EQ(node->getOriginalInputPrecisionAtPort(0), precision) << node->getName(); + EXPECT_EQ(node->getOriginalOutputPrecisionAtPort(0), precision) << node->getName(); +} + +} // namespace + +TEST(EnforceInferencePrecisionBF16Test, KeepsFp32IslandBetweenMandatoryBF16Nodes) { + Config config; + config.inferencePrecision = ov::element::bf16; + config.inferencePrecisionSetExplicitly = true; + + auto context = std::make_shared(config, nullptr, false); + Graph graph; + graph.Init(create_model_with_tagged_node(), context); + + expect_node_precision(find_graph_node_by_name(graph, matmul_before_name), ov::element::bf16); + expect_node_precision(find_graph_node_by_name(graph, cumsum_name), ov::element::f32); + expect_node_precision(find_graph_node_by_name(graph, softmax_name), ov::element::f32); + expect_node_precision(find_graph_node_by_name(graph, matmul_after_name), ov::element::bf16); +} diff --git a/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp b/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp new file mode 100644 index 00000000000000..6f1a7030215ae9 --- /dev/null +++ b/src/plugins/intel_cpu/tests/unit/transformations/disable_bf16_comp_cumsum_sin_gen_test.cpp @@ -0,0 +1,203 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp" + +#include + +#include +#include +#include + +#include "openvino/core/model.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/cum_sum.hpp" +#include "openvino/op/interpolate.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/sin.hpp" +#include "openvino/op/transpose.hpp" +#include "openvino/pass/manager.hpp" +#include "transformations/rt_info/disable_precision_conversion.hpp" + +using namespace testing; +using namespace ov::intel_cpu; + +namespace { + +// Names used to look up matched nodes after the pass has run. +const std::string name_transpose_pre = "transpose_pre"; +const std::string name_interp_pre = "interp_pre"; +const std::string name_transpose_pre_cumsum = "transpose_pre_cumsum"; +const std::string name_cumsum = "cumsum"; +const std::string name_mul_after_cumsum = "mul_after_cumsum"; +const std::string name_transpose_after_mul = "transpose_after_mul"; +const std::string name_scale_mul = "scale_mul"; +const std::string name_interp_after = "interp_after"; +const std::string name_transpose_after_interp = "transpose_after_interp"; +const std::string name_sin = "sin"; + +ov::op::util::InterpolateBase::InterpolateAttrs make_interp_attrs() { + ov::op::util::InterpolateBase::InterpolateAttrs attrs; + attrs.mode = ov::op::util::InterpolateBase::InterpolateMode::LINEAR; + attrs.shape_calculation_mode = ov::op::util::InterpolateBase::ShapeCalcMode::SCALES; + attrs.nearest_mode = ov::op::util::InterpolateBase::NearestMode::ROUND_PREFER_FLOOR; + return attrs; +} + +// Transpose -> Interpolate -> Transpose -> CumSum +// -> Multiply -> Transpose -> Multiply -> Interpolate -> Transpose -> Sin. +std::shared_ptr create_full_chain_model() { + auto input = std::make_shared(ov::element::f32, ov::PartialShape{1, 32}); + + auto order_pre = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_pre = std::make_shared(input, order_pre); + transpose_pre->set_friendly_name(name_transpose_pre); + + auto sizes_pre = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {64L, 1L}); + auto scales_pre = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, {2.0f, 1.0f}); + auto interp_pre = + std::make_shared(transpose_pre, sizes_pre, scales_pre, make_interp_attrs()); + interp_pre->set_friendly_name(name_interp_pre); + + auto order_pre_cumsum = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_pre_cumsum = std::make_shared(interp_pre, order_pre_cumsum); + transpose_pre_cumsum->set_friendly_name(name_transpose_pre_cumsum); + + auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}); + auto cumsum = std::make_shared(transpose_pre_cumsum, axis); + cumsum->set_friendly_name(name_cumsum); + + auto mul_after_cumsum_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1, 64}, {6.2832f}); + auto mul_after_cumsum = std::make_shared(cumsum, mul_after_cumsum_const); + mul_after_cumsum->set_friendly_name(name_mul_after_cumsum); + + auto order_after_mul = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_after_mul = std::make_shared(mul_after_cumsum, order_after_mul); + transpose_after_mul->set_friendly_name(name_transpose_after_mul); + + auto scale_mul_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{64, 1}, {1.0f}); + auto scale_mul = std::make_shared(transpose_after_mul, scale_mul_const); + scale_mul->set_friendly_name(name_scale_mul); + + auto sizes_after = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {128L, 1L}); + auto scales_after = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, {2.0f, 1.0f}); + auto interp_after = + std::make_shared(scale_mul, sizes_after, scales_after, make_interp_attrs()); + interp_after->set_friendly_name(name_interp_after); + + auto order_after_interp = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_after_interp = std::make_shared(interp_after, order_after_interp); + transpose_after_interp->set_friendly_name(name_transpose_after_interp); + + auto sin = std::make_shared(transpose_after_interp); + sin->set_friendly_name(name_sin); + + return std::make_shared(ov::OutputVector{sin}, ov::ParameterVector{input}); +} + +// Same downstream chain but missing the second Multiply between the two +// Transposes — must not match. +std::shared_ptr create_model_missing_scale_mul() { + auto input = std::make_shared(ov::element::f32, ov::PartialShape{1, 32}); + + auto order_pre = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_pre = std::make_shared(input, order_pre); + transpose_pre->set_friendly_name(name_transpose_pre); + + auto sizes_pre = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {64L, 1L}); + auto scales_pre = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, {2.0f, 1.0f}); + auto interp_pre = + std::make_shared(transpose_pre, sizes_pre, scales_pre, make_interp_attrs()); + interp_pre->set_friendly_name(name_interp_pre); + + auto order_pre_cumsum = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_pre_cumsum = std::make_shared(interp_pre, order_pre_cumsum); + transpose_pre_cumsum->set_friendly_name(name_transpose_pre_cumsum); + + auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}); + auto cumsum = std::make_shared(transpose_pre_cumsum, axis); + cumsum->set_friendly_name(name_cumsum); + + auto mul_after_cumsum_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1, 64}, {6.2832f}); + auto mul_after_cumsum = std::make_shared(cumsum, mul_after_cumsum_const); + mul_after_cumsum->set_friendly_name(name_mul_after_cumsum); + + auto order_after_mul = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_after_mul = std::make_shared(mul_after_cumsum, order_after_mul); + transpose_after_mul->set_friendly_name(name_transpose_after_mul); + + // Missing scale Multiply — Interpolate is fed directly by the Transpose. + auto sizes_after = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {64L, 1L}); + auto scales_after = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{2}, {2.0f, 1.0f}); + auto interp_after = + std::make_shared(transpose_after_mul, sizes_after, scales_after, make_interp_attrs()); + interp_after->set_friendly_name(name_interp_after); + + auto order_after_interp = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{2}, {1, 0}); + auto transpose_after_interp = std::make_shared(interp_after, order_after_interp); + transpose_after_interp->set_friendly_name(name_transpose_after_interp); + + auto sin = std::make_shared(transpose_after_interp); + sin->set_friendly_name(name_sin); + + return std::make_shared(ov::OutputVector{sin}, ov::ParameterVector{input}); +} + +void run_test(const std::shared_ptr& model, + const std::unordered_map& expected_bf16_disabled_status) { + ov::pass::Manager manager; + manager.register_pass(); + manager.run_passes(model); + + for (const auto& op : model->get_ops()) { + auto it = expected_bf16_disabled_status.find(op->get_friendly_name()); + if (it == expected_bf16_disabled_status.end()) { + continue; + } + if (it->second) { + ASSERT_TRUE(ov::is_conversion_disabled(op, ov::element::f32, ov::element::bf16)) + << "BF16 conversion is not disabled for node: " << op->get_friendly_name(); + } else { + ASSERT_FALSE(ov::is_conversion_disabled(op, ov::element::f32, ov::element::bf16)) + << "BF16 conversion is unexpectedly disabled for node: " << op->get_friendly_name(); + } + } +} + +} // namespace + +TEST(TransformationTests, DisableBF16CompCumSumSinGen_Positive) { + auto model = create_full_chain_model(); + // Core matched nodes are marked as disabled for BF16 conversion. + std::unordered_map expected_status = { + {name_transpose_pre, true}, + {name_interp_pre, true}, + {name_transpose_pre_cumsum, true}, + {name_cumsum, true}, + {name_mul_after_cumsum, true}, + {name_transpose_after_mul, true}, + {name_scale_mul, true}, + {name_interp_after, true}, + {name_transpose_after_interp, true}, + {name_sin, true}, + }; + run_test(model, expected_status); +} + +TEST(TransformationTests, DisableBF16CompCumSumSinGen_MissingScaleMultiply_NoOp) { + auto model = create_model_missing_scale_mul(); + std::unordered_map expected_status = { + {name_transpose_pre, false}, + {name_interp_pre, false}, + {name_transpose_pre_cumsum, false}, + {name_cumsum, false}, + {name_mul_after_cumsum, false}, + {name_transpose_after_mul, false}, + {name_interp_after, false}, + {name_transpose_after_interp, false}, + {name_sin, false}, + }; + run_test(model, expected_status); +}