Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/plugins/intel_cpu/src/graph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2092,6 +2092,19 @@ void Graph::EnforceInferencePrecision() const {

// Pattern-based node skipping for BF16 precision enforcement
if (inferPrec == ov::element::bf16) {
// Preserve a continuous f32 region around model-declared precision-sensitive operations.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For Rope markup in openvino/src/common/transformations/src/transformations/common_optimizations/mark_rope_input_to_keep_in_mixed_precision.cpp, we just use disable_conversion in transformation level why do we need to add a search logic here ?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The disable_conversion is only a marker, while precision enforcement happens later on CPU plugin graph nodes. Without backward/forward closure from marked seeds, the FP32 region may become discontinuous after graph rewrites and optimizers, causing unintended BF16 enforcement inside the intended sensitive chain.

for (const auto& node : graphNodes) {
if (!node->isBF16ConversionDisabled()) {
continue;
}
const auto inserted = nodesToSkip.insert(node);
if (!inserted.second) {
continue;
}
backwardSkipSearch(node, nodesToSkip);
forwardSkipSearch(node, nodesToSkip);
}
Comment thread
mangguo321 marked this conversation as resolved.
Comment thread
mangguo321 marked this conversation as resolved.

for (const auto& node : graphNodes) {
// Pattern 1: MatMul with Convert from integer to floating point on any input. This basically means that
// converting such an integer input to bf16 leads to loosing accuracy, as bf16 can only exactly represent
Expand Down
1 change: 1 addition & 0 deletions src/plugins/intel_cpu/src/node.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ Node::Node(const std::shared_ptr<ov::Node>& op, GraphContext::CPtr ctx, const Sh
if (is_conversion_disabled(op, element::f16)) {
keepOriginalPrecision = true;
}
disableBF16Conversion = is_conversion_disabled(op, element::f32, element::bf16);
}

Node::Node(const std::string& type,
Expand Down
4 changes: 4 additions & 0 deletions src/plugins/intel_cpu/src/node.h
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,9 @@ class Node {
bool keepOrigPrecision() const {
return keepOriginalPrecision;
}
bool isBF16ConversionDisabled() const {
return disableBF16Conversion;
}

protected:
bool canFuseSimpleOperation(const NodePtr& node) const;
Expand Down Expand Up @@ -721,6 +724,7 @@ class Node {
MemoryFormatFilter memoryFormatFilter;
bool enforceBF16evenForGraphTail = false;
bool keepOriginalPrecision = false;
bool disableBF16Conversion = false;

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

quite strange flag that looks like WA, not the full solution. What if I want to disable conversion from other type

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would not consider this a workaround, but rather an extension of the existing precision-control mechanism in the CPU plugin. We already have a dedicated guard for fp16 (keepOriginalPrecision), so introducing a BF16 specific guard is consistent with current plugin design.


std::string originalLayers; // contains names of the original layers separated by comma

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright (C) 2018-2026 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "disable_bf16_comp_cumsum_sin_gen.hpp"

#include <memory>
#include <vector>

#include "openvino/cc/pass/itt.hpp"
#include "openvino/core/node.hpp"
#include "openvino/core/node_output.hpp"
#include "openvino/core/type/element_type.hpp"
#include "openvino/op/cum_sum.hpp"
#include "openvino/op/interpolate.hpp"
#include "openvino/op/multiply.hpp"
#include "openvino/op/sin.hpp"
#include "openvino/op/transpose.hpp"
#include "openvino/pass/matcher_pass.hpp"
#include "openvino/pass/pattern/matcher.hpp"
#include "openvino/pass/pattern/op/label.hpp"
#include "openvino/pass/pattern/op/or.hpp"
#include "openvino/pass/pattern/op/pattern.hpp"
#include "openvino/pass/pattern/op/wrap_type.hpp"
#include "transformations/rt_info/disable_precision_conversion.hpp"

namespace ov::intel_cpu {

DisableBF16CompCumSumSinGen::DisableBF16CompCumSumSinGen() {
MATCHER_SCOPE(DisableBF16CompCumSumSinGen);
using namespace ov::pass::pattern;
using ov::pass::operator|;

auto interpolate_variations = [](const ov::Output<ov::Node>& input) {
auto interp_v0_m = wrap_type<ov::op::v0::Interpolate>({input, any_input()});
auto interp_v4_m = wrap_type<ov::op::v4::Interpolate>({input, any_input(), any_input()});
auto interp_v4_with_axes_m = wrap_type<ov::op::v4::Interpolate>({input, any_input(), any_input(), any_input()});
auto interp_v11_m = wrap_type<ov::op::v11::Interpolate>({input, any_input()});
auto interp_v11_with_axes_m = wrap_type<ov::op::v11::Interpolate>({input, any_input(), any_input()});
return interp_v0_m | interp_v4_m | interp_v4_with_axes_m | interp_v11_m | interp_v11_with_axes_m;
};

auto transpose_pre_m = wrap_type<ov::op::v1::Transpose>({any_input(), any_input()});
auto interp_pre_m = interpolate_variations(transpose_pre_m);

auto transpose1_m = wrap_type<ov::op::v1::Transpose>({interp_pre_m, any_input()});
auto cumsum_m = wrap_type<ov::op::v0::CumSum>({transpose1_m, any_input()});
auto mul1_m = wrap_type<ov::op::v1::Multiply>({cumsum_m, any_input()});
auto transpose2_m = wrap_type<ov::op::v1::Transpose>({mul1_m, any_input()});
auto mul2_m = wrap_type<ov::op::v1::Multiply>({transpose2_m, any_input()});

auto interp_down_m = interpolate_variations(mul2_m);

auto transpose3_m = wrap_type<ov::op::v1::Transpose>({interp_down_m, any_input()});
auto sin_m = wrap_type<ov::op::v0::Sin>({transpose3_m});

ov::matcher_pass_callback callback = [=](Matcher& m) {
const auto& pattern_map = m.get_pattern_value_map();

auto sin_node = pattern_map.at(sin_m).get_node_shared_ptr();
if (transformation_callback(sin_node)) {
return false;
}

std::vector<std::shared_ptr<ov::Node>> to_mark{
pattern_map.at(transpose_pre_m).get_node_shared_ptr(),
pattern_map.at(interp_pre_m).get_node_shared_ptr(),
pattern_map.at(transpose1_m).get_node_shared_ptr(),
pattern_map.at(cumsum_m).get_node_shared_ptr(),
pattern_map.at(mul1_m).get_node_shared_ptr(),
pattern_map.at(transpose2_m).get_node_shared_ptr(),
pattern_map.at(mul2_m).get_node_shared_ptr(),
pattern_map.at(interp_down_m).get_node_shared_ptr(),
pattern_map.at(transpose3_m).get_node_shared_ptr(),
sin_node,
};
Comment thread
mangguo321 marked this conversation as resolved.

for (const auto& node : to_mark) {
if (node) {
ov::disable_conversion(node, ov::element::f32, ov::element::bf16);
}
}

return true;
};

auto m = std::make_shared<Matcher>(sin_m, matcher_name);
this->register_matcher(m, callback);
}

} // namespace ov::intel_cpu
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright (C) 2018-2026 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#pragma once

#include "openvino/pass/matcher_pass.hpp"

namespace ov::intel_cpu {

/**
* @brief Keeps the F0 oscillator chain (StyleTTS2 / iSTFTNet's `l_sin_gen`)
* in fp32 when the CPU plugin infers in BF16. The chain integrates
* phase through CumSum and produces a periodic excitation via Sin;
* any BF16 rounding along the phase path drifts the accumulated
* phase and audibly distorts the vocoder output.
*
* The pass matches the fixed l_sin_gen core topology below and directly marks
* the matched core nodes with @ref ov::disable_conversion(node, f32, bf16).
* CPU's EnforceInferencePrecision then expands this into a continuous fp32
* island (including the upstream phase-preparation ops) during precision
* enforcement.
*
* Transpose -> Interpolate -> Transpose -> CumSum
* -> Multiply -> Transpose -> Multiply
* -> Interpolate -> Transpose -> Sin
*
* The pass is intentionally scoped to nodes with `l_sin_gen` in friendly
* names to avoid matching unrelated structurally similar chains.
*
*/
class DisableBF16CompCumSumSinGen : public ov::pass::MatcherPass {
public:
OPENVINO_MATCHER_PASS_RTTI("DisableBF16CompCumSumSinGen");
DisableBF16CompCumSumSinGen();
};

} // namespace ov::intel_cpu
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@
#include "transformations/low_precision/mark_dequantization_subgraph.hpp"

// CPU specific transformations
#include "transformations/cpu_opset/common/pass/disable_bf16_comp_cumsum_sin_gen.hpp"
#include "transformations/cpu_opset/common/pass/insert_convert_after_extension.hpp"
#include "transformations/cpu_opset/common/pass/ngram_fusion.hpp"
#include "transformations/cpu_opset/common/pass/permute_slice_n_interpolation.hpp"
Expand Down Expand Up @@ -594,6 +595,7 @@ void Transformations::PreLpt(const std::vector<ov::element::Type>& defaultPrecis
CPU_REGISTER_PASS_COMMON(manager, ov::pass::AUGRUCellFusion);
CPU_REGISTER_PASS_COMMON(manager, SDPASubgraphFusion);
CPU_REGISTER_PASS_COMMON(manager, ov::pass::GatedDeltaNetFusion);
CPU_REGISTER_PASS_COMMON(manager, ov::intel_cpu::DisableBF16CompCumSumSinGen);
CPU_REGISTER_PASS_COMMON(manager, ov::pass::CommonOptimizations);
CPU_REGISTER_PASS_COMMON(manager, ov::pass::KeepConstPrecision, decompression_precisions, false, true);
CPU_SET_CALLBACK_COMMON(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Copyright (C) 2018-2026 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include <gtest/gtest.h>

#include <memory>
#include <string>
#include <vector>

#include "graph.h"
#include "openvino/core/model.hpp"
#include "openvino/op/constant.hpp"
#include "openvino/op/convert.hpp"
#include "openvino/op/cum_sum.hpp"
#include "openvino/op/matmul.hpp"
#include "openvino/op/parameter.hpp"
#include "openvino/op/result.hpp"
#include "openvino/op/softmax.hpp"
#include "transformations/rt_info/disable_precision_conversion.hpp"

using namespace ov::intel_cpu;

namespace {

const std::string matmul_before_name = "matmul_before";
const std::string cumsum_name = "cumsum";
const std::string softmax_name = "softmax";
const std::string matmul_after_name = "matmul_after";

NodePtr find_graph_node_by_name(const Graph& graph, const std::string& name) {
for (const auto& node : graph.GetNodes()) {
if (node->getName() == name || node->getOriginalLayers().find(name) != std::string::npos) {
return node;
}
}
return nullptr;
}

std::shared_ptr<const ov::Model> create_model_with_tagged_node() {
const auto shape = ov::Shape{1, 32};
const auto weights_shape = ov::Shape{32, 32};
auto input = std::make_shared<ov::op::v0::Parameter>(ov::element::f32, shape);
auto matmul_before =
std::make_shared<ov::op::v0::MatMul>(input,
ov::op::v0::Constant::create(ov::element::f32, weights_shape, {1.0f}));
matmul_before->set_friendly_name(matmul_before_name);

auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1});
auto cumsum = std::make_shared<ov::op::v0::CumSum>(matmul_before, axis);
cumsum->set_friendly_name(cumsum_name);
ov::disable_conversion(cumsum, ov::element::f32, ov::element::bf16);

auto island_softmax = std::make_shared<ov::op::v8::Softmax>(cumsum, 1);
island_softmax->set_friendly_name(softmax_name);

auto matmul_after =
std::make_shared<ov::op::v0::MatMul>(island_softmax,
ov::op::v0::Constant::create(ov::element::f32, weights_shape, {1.0f}));
matmul_after->set_friendly_name(matmul_after_name);
auto output = std::make_shared<ov::op::v0::Convert>(matmul_after, ov::element::bf16);

return std::make_shared<const ov::Model>(ov::ResultVector{std::make_shared<ov::op::v0::Result>(output)},
ov::ParameterVector{input});
}

void expect_node_precision(const NodePtr& node, const ov::element::Type& precision) {
ASSERT_NE(node, nullptr);
EXPECT_EQ(node->getOriginalInputPrecisionAtPort(0), precision) << node->getName();
EXPECT_EQ(node->getOriginalOutputPrecisionAtPort(0), precision) << node->getName();
}

} // namespace

TEST(EnforceInferencePrecisionBF16Test, KeepsFp32IslandBetweenMandatoryBF16Nodes) {
Config config;
config.inferencePrecision = ov::element::bf16;
config.inferencePrecisionSetExplicitly = true;

auto context = std::make_shared<GraphContext>(config, nullptr, false);
Graph graph;
graph.Init(create_model_with_tagged_node(), context);

expect_node_precision(find_graph_node_by_name(graph, matmul_before_name), ov::element::bf16);
expect_node_precision(find_graph_node_by_name(graph, cumsum_name), ov::element::f32);
expect_node_precision(find_graph_node_by_name(graph, softmax_name), ov::element::f32);
expect_node_precision(find_graph_node_by_name(graph, matmul_after_name), ov::element::bf16);
}
Loading
Loading