-
Notifications
You must be signed in to change notification settings - Fork 3.3k
[CPU] Add DisableBF16CompCumSumSinGen transformation #37274
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
1c3c352
7eb47e5
df1e2d8
20307bd
3dcd345
544d87e
a18c78a
3b84acb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -690,6 +690,9 @@ class Node { | |
| bool keepOrigPrecision() const { | ||
| return keepOriginalPrecision; | ||
| } | ||
| bool isBF16ConversionDisabled() const { | ||
| return disableBF16Conversion; | ||
| } | ||
|
|
||
| protected: | ||
| bool canFuseSimpleOperation(const NodePtr& node) const; | ||
|
|
@@ -721,6 +724,7 @@ class Node { | |
| MemoryFormatFilter memoryFormatFilter; | ||
| bool enforceBF16evenForGraphTail = false; | ||
| bool keepOriginalPrecision = false; | ||
| bool disableBF16Conversion = false; | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. quite strange flag that looks like WA, not the full solution. What if I want to disable conversion from other type
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would not consider this a workaround, but rather an extension of the existing precision-control mechanism in the CPU plugin. We already have a dedicated guard for fp16 (keepOriginalPrecision), so introducing a BF16 specific guard is consistent with current plugin design. |
||
|
|
||
| std::string originalLayers; // contains names of the original layers separated by comma | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| // Copyright (C) 2018-2026 Intel Corporation | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| #include "disable_bf16_comp_cumsum_sin_gen.hpp" | ||
|
|
||
| #include <memory> | ||
| #include <vector> | ||
|
|
||
| #include "openvino/cc/pass/itt.hpp" | ||
| #include "openvino/core/node.hpp" | ||
| #include "openvino/core/node_output.hpp" | ||
| #include "openvino/core/type/element_type.hpp" | ||
| #include "openvino/op/cum_sum.hpp" | ||
| #include "openvino/op/interpolate.hpp" | ||
| #include "openvino/op/multiply.hpp" | ||
| #include "openvino/op/sin.hpp" | ||
| #include "openvino/op/transpose.hpp" | ||
| #include "openvino/pass/matcher_pass.hpp" | ||
| #include "openvino/pass/pattern/matcher.hpp" | ||
| #include "openvino/pass/pattern/op/label.hpp" | ||
| #include "openvino/pass/pattern/op/or.hpp" | ||
| #include "openvino/pass/pattern/op/pattern.hpp" | ||
| #include "openvino/pass/pattern/op/wrap_type.hpp" | ||
| #include "transformations/rt_info/disable_precision_conversion.hpp" | ||
|
|
||
| namespace ov::intel_cpu { | ||
|
|
||
| DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() { | ||
| MATCHER_SCOPE(DisableBF16CompCumSumSinGen); | ||
| using namespace ov::pass::pattern; | ||
| using ov::pass::operator|; | ||
|
|
||
| auto interpolate_variations = [](const ov::Output<ov::Node>& input) { | ||
| auto interp_v0_m = wrap_type<ov::op::v0::Interpolate>({input, any_input()}); | ||
| auto interp_v4_m = wrap_type<ov::op::v4::Interpolate>({input, any_input(), any_input()}); | ||
| auto interp_v4_with_axes_m = wrap_type<ov::op::v4::Interpolate>({input, any_input(), any_input(), any_input()}); | ||
| auto interp_v11_m = wrap_type<ov::op::v11::Interpolate>({input, any_input()}); | ||
| auto interp_v11_with_axes_m = wrap_type<ov::op::v11::Interpolate>({input, any_input(), any_input()}); | ||
| return interp_v0_m | interp_v4_m | interp_v4_with_axes_m | interp_v11_m | interp_v11_with_axes_m; | ||
| }; | ||
|
|
||
| auto transpose_pre_m = wrap_type<ov::op::v1::Transpose>({any_input(), any_input()}); | ||
| auto interp_pre_m = interpolate_variations(transpose_pre_m); | ||
|
|
||
| auto transpose1_m = wrap_type<ov::op::v1::Transpose>({interp_pre_m, any_input()}); | ||
| auto cumsum_m = wrap_type<ov::op::v0::CumSum>({transpose1_m, any_input()}); | ||
| auto mul1_m = wrap_type<ov::op::v1::Multiply>({cumsum_m, any_input()}); | ||
| auto transpose2_m = wrap_type<ov::op::v1::Transpose>({mul1_m, any_input()}); | ||
| auto mul2_m = wrap_type<ov::op::v1::Multiply>({transpose2_m, any_input()}); | ||
|
|
||
| auto interp_down_m = interpolate_variations(mul2_m); | ||
|
|
||
| auto transpose3_m = wrap_type<ov::op::v1::Transpose>({interp_down_m, any_input()}); | ||
| auto sin_m = wrap_type<ov::op::v0::Sin>({transpose3_m}); | ||
|
|
||
| ov::matcher_pass_callback callback = [=](Matcher& m) { | ||
| const auto& pattern_map = m.get_pattern_value_map(); | ||
|
|
||
| auto sin_node = pattern_map.at(sin_m).get_node_shared_ptr(); | ||
| if (transformation_callback(sin_node)) { | ||
| return false; | ||
| } | ||
|
|
||
| std::vector<std::shared_ptr<ov::Node>> to_mark{ | ||
| pattern_map.at(transpose_pre_m).get_node_shared_ptr(), | ||
| pattern_map.at(interp_pre_m).get_node_shared_ptr(), | ||
| pattern_map.at(transpose1_m).get_node_shared_ptr(), | ||
| pattern_map.at(cumsum_m).get_node_shared_ptr(), | ||
| pattern_map.at(mul1_m).get_node_shared_ptr(), | ||
| pattern_map.at(transpose2_m).get_node_shared_ptr(), | ||
| pattern_map.at(mul2_m).get_node_shared_ptr(), | ||
| pattern_map.at(interp_down_m).get_node_shared_ptr(), | ||
| pattern_map.at(transpose3_m).get_node_shared_ptr(), | ||
| sin_node, | ||
| }; | ||
|
mangguo321 marked this conversation as resolved.
|
||
|
|
||
| for (const auto& node : to_mark) { | ||
| if (node) { | ||
| ov::disable_conversion(node, ov::element::f32, ov::element::bf16); | ||
| } | ||
| } | ||
|
|
||
| return true; | ||
| }; | ||
|
|
||
| auto m = std::make_shared<Matcher>(sin_m, matcher_name); | ||
| this->register_matcher(m, callback); | ||
| } | ||
|
|
||
| } // namespace ov::intel_cpu | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| // Copyright (C) 2018-2026 Intel Corporation | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| #pragma once | ||
|
|
||
| #include "openvino/pass/matcher_pass.hpp" | ||
|
|
||
| namespace ov::intel_cpu { | ||
|
|
||
| /** | ||
| * @brief Keeps the F0 oscillator chain (StyleTTS2 / iSTFTNet's `l_sin_gen`) | ||
| * in fp32 when the CPU plugin infers in BF16. The chain integrates | ||
| * phase through CumSum and produces a periodic excitation via Sin; | ||
| * any BF16 rounding along the phase path drifts the accumulated | ||
| * phase and audibly distorts the vocoder output. | ||
| * | ||
| * The pass matches the fixed l_sin_gen core topology below and directly marks | ||
| * the matched core nodes with @ref ov::disable_conversion(node, f32, bf16). | ||
| * CPU's EnforceInferencePrecision then expands this into a continuous fp32 | ||
| * island (including the upstream phase-preparation ops) during precision | ||
| * enforcement. | ||
| * | ||
| * Transpose -> Interpolate -> Transpose -> CumSum | ||
| * -> Multiply -> Transpose -> Multiply | ||
| * -> Interpolate -> Transpose -> Sin | ||
| * | ||
| * The pass is intentionally scoped to nodes with `l_sin_gen` in friendly | ||
| * names to avoid matching unrelated structurally similar chains. | ||
| * | ||
| */ | ||
| class DisableBF16CompCumSumSinGen : public ov::pass::MatcherPass { | ||
| public: | ||
| OPENVINO_MATCHER_PASS_RTTI("DisableBF16CompCumSumSinGen"); | ||
| DisableBF16CompCumSumSinGen(); | ||
| }; | ||
|
|
||
| } // namespace ov::intel_cpu |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| // Copyright (C) 2018-2026 Intel Corporation | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| #include <gtest/gtest.h> | ||
|
|
||
| #include <memory> | ||
| #include <string> | ||
| #include <vector> | ||
|
|
||
| #include "graph.h" | ||
| #include "openvino/core/model.hpp" | ||
| #include "openvino/op/constant.hpp" | ||
| #include "openvino/op/convert.hpp" | ||
| #include "openvino/op/cum_sum.hpp" | ||
| #include "openvino/op/matmul.hpp" | ||
| #include "openvino/op/parameter.hpp" | ||
| #include "openvino/op/result.hpp" | ||
| #include "openvino/op/softmax.hpp" | ||
| #include "transformations/rt_info/disable_precision_conversion.hpp" | ||
|
|
||
| using namespace ov::intel_cpu; | ||
|
|
||
| namespace { | ||
|
|
||
| const std::string matmul_before_name = "matmul_before"; | ||
| const std::string cumsum_name = "cumsum"; | ||
| const std::string softmax_name = "softmax"; | ||
| const std::string matmul_after_name = "matmul_after"; | ||
|
|
||
| NodePtr find_graph_node_by_name(const Graph& graph, const std::string& name) { | ||
| for (const auto& node : graph.GetNodes()) { | ||
| if (node->getName() == name || node->getOriginalLayers().find(name) != std::string::npos) { | ||
| return node; | ||
| } | ||
| } | ||
| return nullptr; | ||
| } | ||
|
|
||
| std::shared_ptr<const ov::Model> create_model_with_tagged_node() { | ||
| const auto shape = ov::Shape{1, 32}; | ||
| const auto weights_shape = ov::Shape{32, 32}; | ||
| auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, shape); | ||
| auto matmul_before = | ||
| std::make_shared<ov::op::v0::MatMul>(input, | ||
| ov::op::v0::Constant::create(ov::element::f32, weights_shape, {1.0f})); | ||
| matmul_before->set_friendly_name(matmul_before_name); | ||
|
|
||
| auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); | ||
| auto cumsum = std::make_shared<ov::op::v0::CumSum>(matmul_before, axis); | ||
| cumsum->set_friendly_name(cumsum_name); | ||
| ov::disable_conversion(cumsum, ov::element::f32, ov::element::bf16); | ||
|
|
||
| auto island_softmax = std::make_shared<ov::op::v8::Softmax>(cumsum, 1); | ||
| island_softmax->set_friendly_name(softmax_name); | ||
|
|
||
| auto matmul_after = | ||
| std::make_shared<ov::op::v0::MatMul>(island_softmax, | ||
| ov::op::v0::Constant::create(ov::element::f32, weights_shape, {1.0f})); | ||
| matmul_after->set_friendly_name(matmul_after_name); | ||
| auto output = std::make_shared<ov::op::v0::Convert>(matmul_after, ov::element::bf16); | ||
|
|
||
| return std::make_shared<const ov::Model>(ov::ResultVector{std::make_shared<ov::op::v0::Result>(output)}, | ||
| ov::ParameterVector{input}); | ||
| } | ||
|
|
||
| void expect_node_precision(const NodePtr& node, const ov::element::Type& precision) { | ||
| ASSERT_NE(node, nullptr); | ||
| EXPECT_EQ(node->getOriginalInputPrecisionAtPort(0), precision) << node->getName(); | ||
| EXPECT_EQ(node->getOriginalOutputPrecisionAtPort(0), precision) << node->getName(); | ||
| } | ||
|
|
||
| } // namespace | ||
|
|
||
| TEST(EnforceInferencePrecisionBF16Test, KeepsFp32IslandBetweenMandatoryBF16Nodes) { | ||
| Config config; | ||
| config.inferencePrecision = ov::element::bf16; | ||
| config.inferencePrecisionSetExplicitly = true; | ||
|
|
||
| auto context = std::make_shared<GraphContext>(config, nullptr, false); | ||
| Graph graph; | ||
| graph.Init(create_model_with_tagged_node(), context); | ||
|
|
||
| expect_node_precision(find_graph_node_by_name(graph, matmul_before_name), ov::element::bf16); | ||
| expect_node_precision(find_graph_node_by_name(graph, cumsum_name), ov::element::f32); | ||
| expect_node_precision(find_graph_node_by_name(graph, softmax_name), ov::element::f32); | ||
| expect_node_precision(find_graph_node_by_name(graph, matmul_after_name), ov::element::bf16); | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For Rope markup in
openvino/src/common/transformations/src/transformations/common_optimizations/mark_rope_input_to_keep_in_mixed_precision.cpp,we just usedisable_conversionin transformation level why do we need to add a search logic here ?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
disable_conversionis only a marker, while precision enforcement happens later on CPU plugin graph nodes. Without backward/forward closure from marked seeds, the FP32 region may become discontinuous after graph rewrites and optimizers, causing unintended BF16 enforcement inside the intended sensitive chain.