From c57ac9405e1fb8da3bcba9ce26cecfa1ff32664f Mon Sep 17 00:00:00 2001 From: "hrushit.kakadia" Date: Tue, 28 Jul 2026 13:26:44 +0530 Subject: [PATCH 1/3] rebased --- .../fullyconnected_implementations.cpp | 26 +- .../executors/kleidiai/kleidiai_common.hpp | 499 +++++++++++++++++- .../nodes/executors/kleidiai/kleidiai_mm.cpp | 221 ++++---- .../nodes/executors/kleidiai/kleidiai_mm.hpp | 14 + .../intel_cpu/src/nodes/fullyconnected.cpp | 26 +- .../src/arm/matmul_weights_decompression.cpp | 21 + 6 files changed, 700 insertions(+), 107 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp b/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp index a0b5cdc04cad..64ad1253aa5a 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp @@ -101,11 +101,23 @@ static const TypeMapping aclFCTypeMapping { static const TypeMapping aclLowpFCTypeMapping { // {src, wei, bia, dst} pt - {{_u8, _i8, _i32 | _dynamic, _u8}, {bypass(), bypass(), bypass(), bypass()}}, - {{_i8, _i8, _i32 | _dynamic, _i8}, {bypass(), bypass(), bypass(), bypass()}}, {{_u8 | _i8, _i8, _any, _f32}, {bypass(), bypass(), use<3>(), bypass()}} }; +#if defined(OV_CPU_WITH_KLEIDIAI) +static const TypeMapping kleidiaiFCTypeMapping { + // {src, wei, bia, dst} pt + // f32 weights are used as-is by the F32 NEON MLA kernel + {{_f32, _f32, _any, _any}, {bypass(), bypass(), use<0>(), use<0>()}}, + // Keep low-precision (compressed) weights intact so the dynamic-quant + // KleidiAI kernels (i8 / i4 / u4, incl. group asymmetric) receive the + // original weight precision instead of an upconverted f32 tensor. + {{_f32, _i8 | _i4 | _u4, _any, _any}, {bypass(), bypass(), use<0>(), use<0>()}}, + // fallback + {{_any, _any, _any, _any}, {just(), just(), just(), just()}}, +}; +#endif + static const MappingNotation fcMappingNotation { {ARG_SRC, 0}, {ARG_WEI, 1}, @@ -371,14 +383,20 @@ const std::vector>& getImplementations() { VERIFY(noPostOps(config), UNSUPPORTED_POST_OPS); VERIFY(noSparseDecompression(config), UNSUPPORTED_SPARSE_WEIGHTS); VERIFY(all_of(f32, srcType(config), dstType(config)), UNSUPPORTED_SRC_PRECISIONS); - VERIFY(any_of(weiType(config), f32, i8, i4), UNSUPPORTED_WEI_PRECISIONS); + VERIFY(any_of(weiType(config), f32, i8, i4, u4), UNSUPPORTED_WEI_PRECISIONS); VERIFY(implication(hasBias(config), biaType(config) == f32), UNSUPPORTED_SRC_PRECISIONS); VERIFY(weiRank(config) == 2U, UNSUPPORTED_WEI_RANK); VERIFY(MatMulKleidiAIExecutor::supports(config), UNSUPPORTED_BY_EXECUTOR); return true; }, - HasNoOptimalConfig{}, + // createOptimalConfig + [](const FCConfig& config) -> std::optional> { + return createOptimalConfigCommon(config, + kleidiaiFCTypeMapping, + dnnlFCLayoutConfig, + fcMappingNotation); + }, AcceptsAnyShape, CreateDefault{} ) diff --git a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp index 79e134609e6d..f76b37a22c4d 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp +++ b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp @@ -57,11 +57,16 @@ #include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_qsi4c32p_qsu4c32s1s0.h" #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0.h" -namespace ov::intel_cpu::kai_common { - -inline constexpr float FLOAT_MAX = std::numeric_limits::max(); -inline constexpr float FLOAT_MIN = (-std::numeric_limits::max()); +#define FLOAT_MAX std::numeric_limits::max() +#define FLOAT_MIN (-std::numeric_limits::max()) +// Headers for INT4 group asymmetric KAI kernels +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm.h" +#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p_qai4c32p_interface.h" +#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32pscalef32_f32_neon.h" +#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon.h" +namespace ov::intel_cpu::kai_common { enum class KAIKernelTag : std::uint8_t { F32_NEON_MLA, I8_NEON_DOTPROD, @@ -69,13 +74,16 @@ enum class KAIKernelTag : std::uint8_t { I4_NEON_DOTPROD, I4_NEON_IMM, I4_NEON_IMM_GROUP, - I4_NEON_DOTPROD_GROUP + I4_NEON_DOTPROD_GROUP, + I4_NEON_IMM_GROUP_ASYM, + I4_NEON_DOTPROD_GROUP_ASYM }; using KernelInterface = std::variant; + kai_matmul_clamp_f32_qai8dxp_qsi4c32p_ukernel, + kai_matmul_clamp_f32_qsi8d32p_qai4c32p_ukernel>; class uKernelBase { protected: @@ -1181,4 +1189,481 @@ class uKernel : public uKernelBase { }); } // end of execute()... }; -} // namespace ov::intel_cpu::kai_common \ No newline at end of file +template <> +class uKernel : public uKernelBase { +private: + static constexpr kai_matmul_clamp_f32_qsi8d32p_qai4c32p_ukernel uKernelInterface{ + kai_get_m_step_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_n_step_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_mr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_nr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod, + kai_run_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod}; + + MemoryPtr rhsPackedMem; + MemoryPtr& lhsPackedMem; + MemoryCPtr rhsZeroPointsMem; + size_t group_size = 0; + +public: + uKernel(size_t N, size_t K, MemoryPtr& lhsPackedMem, const MemoryArgs& memory) + : lhsPackedMem(lhsPackedMem) { + this->N = N; + this->K = K; + const auto scalesDims = memory.at(ARG_WEI | ARG_ATTR_SCALES)->getDesc().getShape().getDims(); + OPENVINO_ASSERT(scalesDims.size() > 1, "Group quantization requires scales with at least two dimensions. Got ", scalesDims.size(), "."); + OPENVINO_ASSERT(scalesDims[1] != 0 && K % scalesDims[1] == 0, "Invalid scales shape for group quantization."); + group_size = K / scalesDims[1]; + OPENVINO_ASSERT(group_size % 32 == 0, "Group size must be a multiple of 32. Got ", group_size); + rhsZeroPointsMem = memory.at(ARG_WEI | ARG_ATTR_ZERO_POINTS); + OPENVINO_ASSERT(rhsZeroPointsMem, "INT4 asymmetric group quantization requires weight zero-points."); + + const auto zeroPointDims = rhsZeroPointsMem->getDesc().getShape().getDims(); + + OPENVINO_ASSERT(zeroPointDims == scalesDims, "Zero-point and scale tensors must have the same shape."); + const auto zpPrec = rhsZeroPointsMem->getDescPtr()->getPrecision(); + OPENVINO_ASSERT(zpPrec == ov::element::u4 || zpPrec == ov::element::f32, + "INT4 asymmetric group KleidiAI kernel expects u4 or f32 " + "zero-points. Got ", zpPrec, "."); + this->BLOCK_SIZE = uKernelInterface.get_m_step(); + this->mr = uKernelInterface.get_mr(); + this->nr = uKernelInterface.get_nr(); + this->kr = uKernelInterface.get_kr(); + this->sr = uKernelInterface.get_sr(); + } + + size_t get_rhsPackedSize() override { + return kai_get_rhs_packed_size_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon( + N, + K, + nr, + kr, + group_size); + } + + void packData(bool isTransposed, + MemoryCPtr weightsMemory, + MemoryPtr biasMem, + bool hasBias, + float* rhs_scales, + MemoryPtr rhsPackedMemory) override { + OPENVINO_ASSERT(rhs_scales != nullptr, + "INT4 asymmetric group quantization requires scales."); + + OPENVINO_ASSERT(!isTransposed, + "ASYM qai4c32p path currently supports non-transposed NxK weights."); + + OPENVINO_ASSERT(rhsZeroPointsMem, + "INT4 asymmetric group quantization requires zero-points."); + + rhsPackedMem = rhsPackedMemory; + + const auto* bias_ptr = hasBias ? biasMem->getDataAs() : nullptr; + + const size_t groups_per_row = K / group_size; + const size_t num_zero_points = N * groups_per_row; + const size_t weight_bytes = N * K / 2; + + const auto* rhs_qau4_s1s0 = weightsMemory->getDataAs(); + + std::vector rhs_qau4_s0s1(weight_bytes); + for (size_t i = 0; i < weight_bytes; ++i) { + const uint8_t b = rhs_qau4_s1s0[i]; + + rhs_qau4_s0s1[i] = static_cast( + ((b & 0x0FU) << 4) | + ((b & 0xF0U) >> 4)); + } + + // Read a logical U4 value from OpenVINO's packed s1s0 storage. + auto read_u4_s1s0 = [](const uint8_t* data, size_t index) -> uint8_t { + const uint8_t b = data[index / 2]; + + return (index & 1U) ? ((b >> 4) & 0x0FU) + : (b & 0x0FU); + }; + + const auto* zp_qau4_s1s0 = rhsZeroPointsMem->getDataAs(); + + std::vector zero_points_f32(num_zero_points); + + for (size_t i = 0; i < num_zero_points; ++i) { + const int32_t zp_signed = + static_cast(read_u4_s1s0(zp_qau4_s1s0, i)) - 8; + + zero_points_f32[i] = + -static_cast(zp_signed) * rhs_scales[i]; + } + + kai_rhs_pack_nxk_qai4c32p_params params{}; + params.lhs_zero_point = 1; + params.rhs_zero_point = 8; + + kai_run_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon( + 1, + N, + K, + nr, + kr, + sr, + group_size, + rhs_qau4_s0s1.data(), + zero_points_f32.data(), + bias_ptr, + rhs_scales, + rhsPackedMem->getData(), + 0, + ¶ms); + } + KernelInterface getuKernelInterface() override { + return uKernelInterface; + } + + KAIKernelTag getKernelTag() override { + return KAIKernelTag::I4_NEON_DOTPROD_GROUP_ASYM; + } + + size_t getLHSPackedSize(size_t m) override { + const size_t m_blocks = (m + BLOCK_SIZE - 1) / BLOCK_SIZE; + + packedlhs_block_in_bytes = + kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32pscalef32_f32_neon( + BLOCK_SIZE, + K, + group_size, + mr, + kr, + sr); + + return m_blocks * packedlhs_block_in_bytes; + } + + void execute(const ov::intel_cpu::CpuParallelPtr& cpu_parallel, + ov::intel_cpu::Dim M, + ov::intel_cpu::Dim K, + ov::intel_cpu::MemoryPtr dstMem, + ov::intel_cpu::MemoryPtr srcMem) override { + const auto ukernel = + std::get( + getuKernelInterface()); + + auto* lhs = srcMem->getDataAs(); + auto* dst = dstMem->getDataAs(); + + auto* lhs_packed = lhsPackedMem->getDataAs(); + auto* rhs_packed = rhsPackedMem->getDataAs(); + + const size_t m_step = ukernel.get_m_step(); + const size_t n_step = ukernel.get_n_step(); + + const size_t m_blocks = (M + m_step - 1) / m_step; + const size_t n_blocks = (N + n_step - 1) / n_step; + + const size_t lhs_packed_offset = + ukernel.get_lhs_packed_offset(0, K, group_size); + + const size_t lhs_stride = K * sizeof(float); + const size_t dst_stride_row = N * sizeof(float); + const size_t dst_stride_col = sizeof(float); + + ParallelNestingContext nested_context; + + cpu_parallel->parallel_for(m_blocks, [&](size_t m_blk) { + const size_t m_iter = std::min(M - m_blk * m_step, m_step); + + auto* lhs_packed_block = + lhs_packed + m_blk * packedlhs_block_in_bytes; + + kai_run_lhs_quant_pack_qsi8d32pscalef32_f32_neon( + m_iter, + K, + group_size, + mr, + kr, + sr, + 0, + lhs + m_blk * m_step * K, + lhs_stride, + lhs_packed_block); + + cpu_parallel->parallel_for(n_blocks, [&](size_t n_blk) { + const size_t n_start = n_blk * n_step; + const size_t n_iter = std::min(N - n_start, n_step); + + const size_t rhs_packed_offset = + ukernel.get_rhs_packed_offset(n_start, K, group_size); + + const size_t dst_offset = + ukernel.get_dst_offset(m_blk * m_step, + n_start, + dst_stride_row); + + const auto* lhs_ptr = + static_cast(lhs_packed_block + lhs_packed_offset); + + const auto* rhs_ptr = + static_cast(rhs_packed + rhs_packed_offset); + + auto* dst_ptr = + dst + dst_offset / sizeof(float); + + ukernel.run_matmul(m_iter, + n_iter, + K, + group_size, + lhs_ptr, + rhs_ptr, + dst_ptr, + dst_stride_row, + dst_stride_col, + FLOAT_MIN, + FLOAT_MAX); + }); + }); + } +}; + +template<> +class uKernel : public uKernelBase { +private: + static constexpr kai_matmul_clamp_f32_qsi8d32p_qai4c32p_ukernel uKernelInterface{ + kai_get_m_step_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_n_step_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_mr_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_nr_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm, + kai_run_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm}; + + MemoryPtr rhsPackedMem; + MemoryPtr& lhsPackedMem; + MemoryCPtr rhsZeroPointsMem; + size_t group_size = 0; + +public: + uKernel(size_t N, size_t K, MemoryPtr& lhsPackedMem, const MemoryArgs& memory) + : lhsPackedMem(lhsPackedMem) { + this->N = N; + this->K = K; + const auto scalesDims = memory.at(ARG_WEI | ARG_ATTR_SCALES)->getDesc().getShape().getDims(); + OPENVINO_ASSERT(scalesDims.size() > 1, "Group quantization requires scales with at least two dimensions. Got ", scalesDims.size(), "."); + OPENVINO_ASSERT(scalesDims[1] != 0 && K % scalesDims[1] == 0, "Invalid scales shape for group quantization."); + group_size = K / scalesDims[1]; + OPENVINO_ASSERT(group_size % 32 == 0, "Group size must be a multiple of 32. Got ", group_size); + rhsZeroPointsMem = memory.at(ARG_WEI | ARG_ATTR_ZERO_POINTS); + OPENVINO_ASSERT(rhsZeroPointsMem, "INT4 asymmetric group quantization requires weight zero-points."); + + const auto zeroPointDims = rhsZeroPointsMem->getDesc().getShape().getDims(); + + OPENVINO_ASSERT(zeroPointDims == scalesDims, "Zero-point and scale tensors must have the same shape."); + const auto zpPrec = rhsZeroPointsMem->getDescPtr()->getPrecision(); + OPENVINO_ASSERT(zpPrec == ov::element::u4 || zpPrec == ov::element::f32, + "INT4 asymmetric group KleidiAI kernel expects u4 or f32 " + "zero-points. Got ", zpPrec, "."); + this->BLOCK_SIZE = uKernelInterface.get_m_step(); + this->mr = uKernelInterface.get_mr(); + this->nr = uKernelInterface.get_nr(); + this->kr = uKernelInterface.get_kr(); + this->sr = uKernelInterface.get_sr(); + } + + size_t get_rhsPackedSize() override { + return kai_get_rhs_packed_size_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon( + N, + K, + nr, + kr, + group_size); + } + + void packData(bool isTransposed, + MemoryCPtr weightsMemory, + MemoryPtr biasMem, + bool hasBias, + float* rhs_scales, + MemoryPtr rhsPackedMemory) override { + OPENVINO_ASSERT(rhs_scales != nullptr, + "INT4 asymmetric group quantization requires scales."); + + OPENVINO_ASSERT(!isTransposed, + "ASYM qai4c32p path currently supports non-transposed NxK weights."); + + OPENVINO_ASSERT(rhsZeroPointsMem, + "INT4 asymmetric group quantization requires zero-points."); + + rhsPackedMem = rhsPackedMemory; + + const auto* bias_ptr = hasBias ? biasMem->getDataAs() : nullptr; + + const size_t groups_per_row = K / group_size; + const size_t num_zero_points = N * groups_per_row; + const size_t weight_bytes = N * K / 2; + + const auto* rhs_qau4_s1s0 = weightsMemory->getDataAs(); + + std::vector rhs_qau4_s0s1(weight_bytes); + for (size_t i = 0; i < weight_bytes; ++i) { + const uint8_t b = rhs_qau4_s1s0[i]; + + rhs_qau4_s0s1[i] = static_cast( + ((b & 0x0FU) << 4) | + ((b & 0xF0U) >> 4)); + } + + // Read a logical U4 value from OpenVINO's packed s1s0 storage. + auto read_u4_s1s0 = [](const uint8_t* data, size_t index) -> uint8_t { + const uint8_t b = data[index / 2]; + + return (index & 1U) ? ((b >> 4) & 0x0FU) + : (b & 0x0FU); + }; + + const auto* zp_qau4_s1s0 = rhsZeroPointsMem->getDataAs(); + + std::vector zero_points_f32(num_zero_points); + + for (size_t i = 0; i < num_zero_points; ++i) { + const int32_t zp_signed = + static_cast(read_u4_s1s0(zp_qau4_s1s0, i)) - 8; + + zero_points_f32[i] = + -static_cast(zp_signed) * rhs_scales[i]; + } + + kai_rhs_pack_nxk_qai4c32p_params params{}; + params.lhs_zero_point = 1; + params.rhs_zero_point = 8; + + kai_run_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon( + 1, + N, + K, + nr, + kr, + sr, + group_size, + rhs_qau4_s0s1.data(), + zero_points_f32.data(), + bias_ptr, + rhs_scales, + rhsPackedMem->getData(), + 0, + ¶ms); + } + KernelInterface getuKernelInterface() override { + return uKernelInterface; + } + + KAIKernelTag getKernelTag() override { + return KAIKernelTag::I4_NEON_IMM_GROUP_ASYM; + } + + size_t getLHSPackedSize(size_t m) override { + const size_t m_blocks = (m + BLOCK_SIZE - 1) / BLOCK_SIZE; + + packedlhs_block_in_bytes = + kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32pscalef32_f32_neon( + BLOCK_SIZE, + K, + group_size, + mr, + kr, + sr); + + return m_blocks * packedlhs_block_in_bytes; + } + + void execute(const ov::intel_cpu::CpuParallelPtr& cpu_parallel, + ov::intel_cpu::Dim M, + ov::intel_cpu::Dim K, + ov::intel_cpu::MemoryPtr dstMem, + ov::intel_cpu::MemoryPtr srcMem) override { + const auto ukernel = + std::get( + getuKernelInterface()); + + auto* lhs = srcMem->getDataAs(); + auto* dst = dstMem->getDataAs(); + + auto* lhs_packed = lhsPackedMem->getDataAs(); + auto* rhs_packed = rhsPackedMem->getDataAs(); + + const size_t m_step = ukernel.get_m_step(); + const size_t n_step = ukernel.get_n_step(); + + const size_t m_blocks = (M + m_step - 1) / m_step; + const size_t n_blocks = (N + n_step - 1) / n_step; + + const size_t lhs_packed_offset = + ukernel.get_lhs_packed_offset(0, K, group_size); + + const size_t lhs_stride = K * sizeof(float); + const size_t dst_stride_row = N * sizeof(float); + const size_t dst_stride_col = sizeof(float); + + ParallelNestingContext nested_context; + + cpu_parallel->parallel_for(m_blocks, [&](size_t m_blk) { + const size_t m_iter = std::min(M - m_blk * m_step, m_step); + + auto* lhs_packed_block = + lhs_packed + m_blk * packedlhs_block_in_bytes; + + kai_run_lhs_quant_pack_qsi8d32pscalef32_f32_neon( + m_iter, + K, + group_size, + mr, + kr, + sr, + 0, + lhs + m_blk * m_step * K, + lhs_stride, + lhs_packed_block); + + cpu_parallel->parallel_for(n_blocks, [&](size_t n_blk) { + const size_t n_start = n_blk * n_step; + const size_t n_iter = std::min(N - n_start, n_step); + + const size_t rhs_packed_offset = + ukernel.get_rhs_packed_offset(n_start, K, group_size); + + const size_t dst_offset = + ukernel.get_dst_offset(m_blk * m_step, + n_start, + dst_stride_row); + + const auto* lhs_ptr = + static_cast(lhs_packed_block + lhs_packed_offset); + + const auto* rhs_ptr = + static_cast(rhs_packed + rhs_packed_offset); + + auto* dst_ptr = + dst + dst_offset / sizeof(float); + + ukernel.run_matmul(m_iter, + n_iter, + K, + group_size, + lhs_ptr, + rhs_ptr, + dst_ptr, + dst_stride_row, + dst_stride_col, + FLOAT_MIN, + FLOAT_MAX); + }); + }); + } +}; +} // namespace ov::intel_cpu::kai_common diff --git a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp index f21fafd566a4..0d956c138fac 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp @@ -53,23 +53,32 @@ static bool useDynamicQuantizationImpl(const FCAttrs& attrs, const MemoryDescPtr return false; } - return weightDesc->getPrecision() == element::i8 || weightDesc->getPrecision() == element::i4; + return weightDesc->getPrecision() == element::i8 || weightDesc->getPrecision() == element::i4 || weightDesc->getPrecision() == element::u4; } bool MatMulKleidiAIExecutor::supports(const FCConfig& config) { - VERIFY(hasArmISASupport(ArmISA::ASIMD), UNSUPPORTED_ISA); - return config.descs.at(ARG_WEI)->getPrecision() == element::f32 || - useDynamicQuantizationImpl(config.attrs, config.descs.at(ARG_WEI)); + bool returnValue = config.descs.at(ARG_WEI)->getPrecision() == element::f32 || + useDynamicQuantizationImpl(config.attrs, config.descs.at(ARG_WEI)); + return returnValue; } bool MatMulKleidiAIExecutor::isGroupQuantizationEnabled(const MemoryArgs& memory) { auto scales = memory.at(ARG_WEI | ARG_ATTR_SCALES)->getDesc().getShape().getStaticDims(); - if (scales.size() == 1) { - return false; - } + OPENVINO_ASSERT(scales.size() > 1, + "Scales tensor to have at least 2 dimensions. Got ", + scales.size(), + " dimension(s)."); + // std::cout << "Scales[0,1,2] value: " << scales[0] << ", " << scales[1] << ", " << scales[2] << std::endl; return (scales[1] > 1); } +bool MatMulKleidiAIExecutor::isAsymmetricQuantizationEnabled(const MemoryArgs& memory) { + const auto zpIt = memory.find(ARG_WEI | ARG_ATTR_ZERO_POINTS); + return zpIt != memory.end() && + zpIt->second != nullptr && + !zpIt->second->getDesc().empty(); +} + MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, const MemoryArgs& memory, const ExecutorContext::CPtr& context) @@ -98,109 +107,141 @@ MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, const VectorDims wgtDims2D = reshapeDownToRank<2>(wgtDims); originalWeightsDesc = std::make_shared(originalWeightsDesc->getPrecision(), Shape{wgtDims2D}); auto dnnlSrcDesc = MemoryDescUtils::convertToDnnlMemoryDesc(originalWeightsDesc); + + kernelLookupKey = 0; - bool isTransposed = false; - float* rhs_scales = nullptr; - - // Whether dynamic quantization is enabled const bool useDynamicQuant = useDynamicQuantizationImpl(attrs, originalWeightsDesc); - if (!useDynamicQuant) { - _kernel = std::make_shared>(N, K); - - auto dstDesc = originalWeightsDesc->cloneWithNewPrecision(memory.at(ARG_SRC)->getDescPtr()->getPrecision()); - auto dnnlDstDesc = MemoryDescUtils::convertToDnnlMemoryDesc(dstDesc); - - if (!attrs.weightsNonTransposed) { - dnnlDstDesc = acl_fc_executor::makeTransposedWeightDescriptor(dnnlDstDesc, dnnlSrcDesc); - aclfcAttrs.isWeightsRepacked = true; - } - MemoryCPtr packedWeights = - acl_fc_executor::reorderWeights(memory, context, aclfcAttrs, dnnlSrcDesc, dnnlDstDesc); - const size_t rhsPackedSize = _kernel->get_rhsPackedSize(); - auto rhsPackedDesc = std::make_shared(u8, Shape({rhsPackedSize})); - rhsPackedMem = std::make_shared(context->getEngine(), rhsPackedDesc); - - _kernel->packData(false, packedWeights, biasMem, hasBias, nullptr, rhsPackedMem); + kernelLookupKey = kernelLookup::WEIGHT_FP32; } else { - MemoryPtr weightsMemory = memory.at(ARG_WEI); - isTransposed = attrs.weightsNonTransposed; - // Check if weights are in int4 or int8 - if (weightsMemory->getDescPtr()->getPrecision() == element::i4) { - if (isGroupQuantizationEnabled(memory)) { - if (hasArmISASupport(ArmISA::I8MM)) { - _kernel = - std::make_shared>(N, - K, - lhsPackedMem, - memory); - } else { - _kernel = std::make_shared>( - N, - K, - lhsPackedMem, - memory); - } - } else { - if (hasArmISASupport(ArmISA::I8MM)) { - _kernel = - std::make_shared>(N, - K, - lhsPackedMem); - } else { - _kernel = - std::make_shared>(N, - K, - lhsPackedMem); - } - } - const size_t rhsPackedSize = _kernel->get_rhsPackedSize(); - auto rhsPackedDesc = std::make_shared(i8, Shape({rhsPackedSize})); - rhsPackedMem = std::make_shared(context->getEngine(), rhsPackedDesc); + const auto weightPrecision = memory.at(ARG_WEI)->getDescPtr()->getPrecision(); + if (weightPrecision == element::i4 || weightPrecision == element::u4) { + kernelLookupKey |= kernelLookup::WEIGHT_INT4; + } else if (weightPrecision == element::i8) { + kernelLookupKey |= kernelLookup::WEIGHT_INT8; } else { - if (hasArmISASupport(ArmISA::I8MM)) { - _kernel = - std::make_shared>(N, K, lhsPackedMem); - } else { - _kernel = - std::make_shared>(N, - K, - lhsPackedMem); - } + OPENVINO_THROW_NOT_IMPLEMENTED("Unsupported weight format by KleidiAI executor: ", + weightPrecision); + } - if (!attrs.weightsNonTransposed) { - auto dnnlSrcDesc = MemoryDescUtils::convertToDnnlMemoryDesc(originalWeightsDesc); - auto dnnlDstDesc = acl_fc_executor::makeTransposedWeightDescriptor(dnnlSrcDesc, dnnlSrcDesc); - weightsMemory = acl_fc_executor::reorderData(dnnlSrcDesc, dnnlDstDesc, memory.at(ARG_WEI), context); - } - const size_t rhsPackedSize = _kernel->get_rhsPackedSize(); - auto rhsPackedDesc = std::make_shared(i8, Shape({rhsPackedSize})); - rhsPackedMem = std::make_shared(context->getEngine(), rhsPackedDesc); + if (hasArmISASupport(ArmISA::I8MM)) { + kernelLookupKey |= kernelLookup::ISA_I8MM; + } else if (hasArmISASupport(ArmISA::DOTPROD)) { + kernelLookupKey |= kernelLookup::ISA_DOTPROD; + } else { + OPENVINO_THROW_NOT_IMPLEMENTED("KleidiAI quantized kernels require Arm dotprod or i8mm support."); } - rhs_scales = static_cast(memory.at(ARG_WEI | ARG_ATTR_SCALES)->getData()); + kernelLookupKey |= isGroupQuantizationEnabled(memory) ? kernelLookup::QUANT_GROUP + : kernelLookup::QUANT_CHANNEL; + kernelLookupKey |= isAsymmetricQuantizationEnabled(memory) ? kernelLookup::QUANT_ASYMMETRIC + : kernelLookup::QUANT_SYMMETRIC; + } + + const auto createPackedMemory = [&](size_t size, const element::Type& precision) { + auto desc = std::make_shared(precision, Shape({size})); + rhsPackedMem = std::make_shared(context->getEngine(), desc); + }; + const auto packQuantizedWeights = [&](bool isTransposed, MemoryCPtr weightsMemory) { + createPackedMemory(_kernel->get_rhsPackedSize(), i8); + auto* rhsScales = static_cast(memory.at(ARG_WEI | ARG_ATTR_SCALES)->getData()); std::vector transposedScales; - // When the weight's Transpose was elided by the graph optimizer (weightsNonTransposed==true, - // see FuseFCAndTransposeOnWeights), the per-group decompression scales feeding this FC were - // transposed together with the weights (ConvertFullyConnectedToFullyConnectedCompressed:: - // process_compressed_weights) and had that Transpose elided the same way. Their physical layout - // is therefore still [numGroups, N] instead of the canonical [N, numGroups] the KAI group - // kernels expect, so it must be repacked here before use. if (isTransposed && isGroupQuantizationEnabled(memory)) { const auto numGroups = memory.at(ARG_WEI | ARG_ATTR_SCALES)->getDesc().getShape().getStaticDims()[1]; transposedScales.resize(N * numGroups); for (size_t g = 0; g < numGroups; ++g) { for (size_t n = 0; n < N; ++n) { - transposedScales[n * numGroups + g] = rhs_scales[g * N + n]; + transposedScales[n * numGroups + g] = rhsScales[g * N + n]; } } - rhs_scales = transposedScales.data(); + rhsScales = transposedScales.data(); } + _kernel->packData(isTransposed, weightsMemory, biasMem, hasBias, rhsScales, rhsPackedMem); + }; - _kernel->packData(isTransposed, weightsMemory, biasMem, hasBias, rhs_scales, rhsPackedMem); + switch (kernelLookupKey) { + case WEIGHT_FP32: { + _kernel = std::make_shared>(N, K); + + auto dstDesc = originalWeightsDesc->cloneWithNewPrecision(memory.at(ARG_SRC)->getDescPtr()->getPrecision()); + auto dnnlDstDesc = MemoryDescUtils::convertToDnnlMemoryDesc(dstDesc); + if (!attrs.weightsNonTransposed) { + dnnlDstDesc = acl_fc_executor::makeTransposedWeightDescriptor(dnnlDstDesc, dnnlSrcDesc); + aclfcAttrs.isWeightsRepacked = true; + } + + const MemoryCPtr packedWeights = + acl_fc_executor::reorderWeights(memory, context, aclfcAttrs, dnnlSrcDesc, dnnlDstDesc); + createPackedMemory(_kernel->get_rhsPackedSize(), u8); + _kernel->packData(false, packedWeights, biasMem, hasBias, nullptr, rhsPackedMem); + break; + } + + case WEIGHT_INT4 | ISA_I8MM | QUANT_GROUP | QUANT_SYMMETRIC: + _kernel = std::make_shared>( + N, K, lhsPackedMem, memory); + packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); + break; + + case WEIGHT_INT4 | ISA_DOTPROD | QUANT_GROUP | QUANT_SYMMETRIC: + _kernel = std::make_shared>( + N, K, lhsPackedMem, memory); + packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); + break; + + case WEIGHT_INT4 | ISA_I8MM | QUANT_GROUP | QUANT_ASYMMETRIC: + _kernel = std::make_shared>( + N, K, lhsPackedMem, memory); + packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); + break; + + case WEIGHT_INT4 | ISA_DOTPROD | QUANT_GROUP | QUANT_ASYMMETRIC: + _kernel = std::make_shared>( + N, K, lhsPackedMem, memory); + packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); + break; + + case WEIGHT_INT4 | ISA_I8MM | QUANT_CHANNEL | QUANT_SYMMETRIC: + _kernel = std::make_shared>(N, + K, + lhsPackedMem); + packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); + break; + + case WEIGHT_INT4 | ISA_DOTPROD | QUANT_CHANNEL | QUANT_SYMMETRIC: + _kernel = std::make_shared>(N, + K, + lhsPackedMem); + packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); + break; + + case WEIGHT_INT8 | ISA_DOTPROD | QUANT_CHANNEL | QUANT_SYMMETRIC: + case WEIGHT_INT8 | ISA_I8MM | QUANT_CHANNEL | QUANT_SYMMETRIC: { + const bool useI8MM = (kernelLookupKey & ISA_I8MM) != 0; + if (useI8MM) { + _kernel = std::make_shared>( + N, K, lhsPackedMem); + } else { + _kernel = std::make_shared>( + N, K, lhsPackedMem); + } + + MemoryPtr weightsMemory = memory.at(ARG_WEI); + if (!attrs.weightsNonTransposed) { + const auto srcDesc = MemoryDescUtils::convertToDnnlMemoryDesc(originalWeightsDesc); + const auto dstDesc = acl_fc_executor::makeTransposedWeightDescriptor(srcDesc, srcDesc); + weightsMemory = acl_fc_executor::reorderData(srcDesc, dstDesc, memory.at(ARG_WEI), context); + } + packQuantizedWeights(false, weightsMemory); + break; } - // Create scratchpad to initialize memory for LHS in update() + + default: + OPENVINO_THROW_NOT_IMPLEMENTED("Unsupported KleidiAI kernel configuration. Lookup key: ", + kernelLookupKey); + } + scratchPad = context->getScratchPad(); } diff --git a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp index f72ebbb40419..05810282907f 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp +++ b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp @@ -33,6 +33,18 @@ class MatMulKleidiAIExecutor : public Executor { void setKaiExecutorImplAsGatherMatmul(); void set_gather_idx(const std::vector>& idxMap); + enum kernelLookup { + WEIGHT_FP32 = 1, + WEIGHT_INT8 = 1 << 2, + WEIGHT_INT4 = 1 << 3, + ISA_DOTPROD = 1 << 4, + ISA_I8MM = 1 << 5, + QUANT_CHANNEL = 1 << 6, + QUANT_GROUP = 1 << 7, + QUANT_SYMMETRIC = 1 << 8, + QUANT_ASYMMETRIC = 1 << 9 + }; + private: static bool isGroupQuantizationEnabled(const MemoryArgs& memory); // IMPL_TYPE :: Default @@ -40,6 +52,7 @@ class MatMulKleidiAIExecutor : public Executor { // IMPL_TYPE :: GatherMatmul // [B, M, K] -> gather -> [M', K] * [N', K] -> scatter -> [B, N, K] enum class IMPL_TYPE : uint8_t { Default, GatherMatmul }; + static bool isAsymmetricQuantizationEnabled(const MemoryArgs& memory); DnnlScratchPadPtr scratchPad; IMPL_TYPE KaiExecutorImpl = IMPL_TYPE::Default; std::vector> gather_idx; @@ -53,6 +66,7 @@ class MatMulKleidiAIExecutor : public Executor { size_t M = 0UL, N = 0UL, K = 0UL; ExecutorContext::CPtr executorContext; std::shared_ptr _kernel; + size_t kernelLookupKey = 0; }; using MatMulKleidiAIExecutorPtr = std::shared_ptr; diff --git a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp index 89b95bd7d6b3..7a9af1db7ef2 100644 --- a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp +++ b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp @@ -78,7 +78,9 @@ ov::element::TypeVector FullyConnected::getSupportedCompressedWeightsTypes([[may } return supportedDataTypes; #elif defined(OV_CPU_WITH_KLEIDIAI) - return {Type_t::i8, Type_t::i4}; + // Symmetric weight-only compression uses i4. Asymmetric groupwise + // compression exported by Optimum uses u4 weights plus u4 zero-points. + return {Type_t::i8, Type_t::i4, Type_t::u4}; #else return {}; #endif @@ -202,9 +204,23 @@ bool FullyConnected::isSupportedCompressedOperation([[maybe_unused]] const std:: return false; } - if (op->get_input_size() > WEIGHT_ZERO_POINTS && - op->input(WEIGHT_ZERO_POINTS).get_element_type() != ov::element::dynamic) { - return false; + const bool hasWeightZeroPoints = + op->get_input_size() > WEIGHT_ZERO_POINTS && + op->input(WEIGHT_ZERO_POINTS).get_element_type() != ov::element::dynamic; + + if (hasWeightZeroPoints) { + const auto weightsType = op->input(WEIGHTS).get_element_type(); + const auto zeroPointsType = op->input(WEIGHT_ZERO_POINTS).get_element_type(); + // unsigned INT4 weights with unsigned INT4 per-group zero-points. + if (weightsType != ov::element::u4 || zeroPointsType != ov::element::u4 || isNotGroupWise) { + return false; + } + // The asymmetric path requires one zero-point for every weight + // scale entry. This rejects per-tensor and unsupported layouts. + if (shape_size(op->input(WEIGHT_ZERO_POINTS).get_shape()) != + shape_size(op->input(WEIGHT_SCALES).get_shape())) { + return false; + } } } catch (...) { return false; @@ -576,8 +592,6 @@ void FullyConnected::initSupportedPrimitiveDescriptors() { attrs.dynamicQuantizationGroupSize = context->getConfig().fcDynamicQuantizationGroupSize; attrs.modelType = context->getConfig().modelType; - attrs.dqScales = getDQScales(); - attrs.postOps = getPostOps(fusedWith); const auto& srcTypes = getOriginalInputPrecisions(); diff --git a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/arm/matmul_weights_decompression.cpp b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/arm/matmul_weights_decompression.cpp index 93c9e77e592b..410588bd8b68 100644 --- a/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/arm/matmul_weights_decompression.cpp +++ b/src/plugins/intel_cpu/tests/functional/custom/subgraph_tests/src/arm/matmul_weights_decompression.cpp @@ -91,6 +91,27 @@ INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeightsGrp_Kleidiai, ::testing::Values(true)), MatmulWeightsDecompression::getTestCaseName); +const std::vector input_shapes_with_groups = { + {{{}, {{1, 7, 256}}}, {256, 128}, 32lu}, // group_size = 32 + { {{-1, -1, -1}, {{10, 40, 64}, {11, 40, 64}}}, // data_shape + {64, 128}, // weights_shape + 32lu}, // group_size = 16 +}; + +INSTANTIATE_TEST_SUITE_P(smoke_MatMulCompressedWeightsGrp_Kleidiai_asym, + MatmulWeightsDecompression, + ::testing::Combine(::testing::ValuesIn(input_shapes_with_groups), + ::testing::Values(ov::element::u4), + ::testing::ValuesIn(decompression_precisions), + ::testing::Values(ov::element::dynamic), + ::testing::Values(true), + ::testing::Values(DecompressionType::full), // group-wise requires full + ::testing::Values(DecompressionType::full), // asymmetric with zero-point + ::testing::Values(false), + ::testing::Values(enable_dyn_quant_config_kleidiai), + ::testing::ValuesIn(fusing_params), + ::testing::Values(true)), // should_use_decompression_impl() can also be passed + MatmulWeightsDecompression::getTestCaseName); const std::vector weights_precisions = {ov::element::u8, ov::element::i8}; From ed4007848b3d48b69e68c016a7232cd2b76a1314 Mon Sep 17 00:00:00 2001 From: "hrushit.kakadia" Date: Tue, 4 Aug 2026 14:22:50 +0530 Subject: [PATCH 2/3] resolved comments --- .../fullyconnected_implementations.cpp | 2 ++ .../executors/kleidiai/kleidiai_common.hpp | 4 +-- .../nodes/executors/kleidiai/kleidiai_mm.cpp | 14 ++++----- .../intel_cpu/src/nodes/fullyconnected.cpp | 30 ++++++++++++------- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp b/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp index 64ad1253aa5a..2ce2c13f73f2 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/fullyconnected_implementations.cpp @@ -101,6 +101,8 @@ static const TypeMapping aclFCTypeMapping { static const TypeMapping aclLowpFCTypeMapping { // {src, wei, bia, dst} pt + {{_u8, _i8, _i32 | _dynamic, _u8}, {bypass(), bypass(), bypass(), bypass()}}, + {{_i8, _i8, _i32 | _dynamic, _i8}, {bypass(), bypass(), bypass(), bypass()}}, {{_u8 | _i8, _i8, _any, _f32}, {bypass(), bypass(), use<3>(), bypass()}} }; diff --git a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp index f76b37a22c4d..52814f37a892 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp +++ b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp @@ -57,8 +57,6 @@ #include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_qsi4c32p_qsu4c32s1s0.h" #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0.h" -#define FLOAT_MAX std::numeric_limits::max() -#define FLOAT_MIN (-std::numeric_limits::max()) // Headers for INT4 group asymmetric KAI kernels #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p4x4_qai4c32p4x4_8x4_neon_dotprod.h" #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qai4c32p/kai_matmul_clamp_f32_qsi8d32p4x8_qai4c32p4x8_8x4_neon_i8mm.h" @@ -67,6 +65,8 @@ #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon.h" namespace ov::intel_cpu::kai_common { +inline constexpr float FLOAT_MAX = std::numeric_limits::max(); +inline constexpr float FLOAT_MIN = (-std::numeric_limits::max()); enum class KAIKernelTag : std::uint8_t { F32_NEON_MLA, I8_NEON_DOTPROD, diff --git a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp index 0d956c138fac..9bd61c72d4bc 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp @@ -57,18 +57,16 @@ static bool useDynamicQuantizationImpl(const FCAttrs& attrs, const MemoryDescPtr } bool MatMulKleidiAIExecutor::supports(const FCConfig& config) { - bool returnValue = config.descs.at(ARG_WEI)->getPrecision() == element::f32 || - useDynamicQuantizationImpl(config.attrs, config.descs.at(ARG_WEI)); - return returnValue; + VERIFY(hasArmISASupport(ArmISA::ASIMD), UNSUPPORTED_ISA); + return config.descs.at(ARG_WEI)->getPrecision() == element::f32 || + useDynamicQuantizationImpl(config.attrs, config.descs.at(ARG_WEI)); } bool MatMulKleidiAIExecutor::isGroupQuantizationEnabled(const MemoryArgs& memory) { auto scales = memory.at(ARG_WEI | ARG_ATTR_SCALES)->getDesc().getShape().getStaticDims(); - OPENVINO_ASSERT(scales.size() > 1, - "Scales tensor to have at least 2 dimensions. Got ", - scales.size(), - " dimension(s)."); - // std::cout << "Scales[0,1,2] value: " << scales[0] << ", " << scales[1] << ", " << scales[2] << std::endl; + if (scales.size() == 1) { + return false; + } return (scales[1] > 1); } diff --git a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp index 7a9af1db7ef2..8e65cf28ffad 100644 --- a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp +++ b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp @@ -197,10 +197,16 @@ bool FullyConnected::isSupportedCompressedOperation([[maybe_unused]] const std:: return false; } - bool isNotGroupWise = (IC % G != 0 || IC / G < 4 || OC == 1 || (IC / G) % 32 != 0); - bool isNotChannelWise = - (op->get_input_size() > WEIGHT_SCALES && shape_size(op->input(WEIGHT_SCALES).get_shape()) != OC); - if (isNotChannelWise && isNotGroupWise) { + const auto scalesShape = op->input(WEIGHT_SCALES).get_shape(); + const bool isChannelWise = shape_size(scalesShape) == OC; + const bool hasValidGroupGeometry = + IC % G == 0 && + IC / G >= 4 && + OC != 1 && + (IC / G) % 32 == 0; + const bool isGroupWise = !isChannelWise && hasValidGroupGeometry; + + if (!isChannelWise && !isGroupWise) { return false; } @@ -211,14 +217,17 @@ bool FullyConnected::isSupportedCompressedOperation([[maybe_unused]] const std:: if (hasWeightZeroPoints) { const auto weightsType = op->input(WEIGHTS).get_element_type(); const auto zeroPointsType = op->input(WEIGHT_ZERO_POINTS).get_element_type(); - // unsigned INT4 weights with unsigned INT4 per-group zero-points. - if (weightsType != ov::element::u4 || zeroPointsType != ov::element::u4 || isNotGroupWise) { + // KleidiAI supports asymmetric INT4 only for group-wise u4. + if (weightsType != ov::element::u4 || + zeroPointsType != ov::element::u4 || + !isGroupWise) { return false; } - // The asymmetric path requires one zero-point for every weight - // scale entry. This rejects per-tensor and unsupported layouts. - if (shape_size(op->input(WEIGHT_ZERO_POINTS).get_shape()) != - shape_size(op->input(WEIGHT_SCALES).get_shape())) { + + const auto zeroPointsShape = + op->input(WEIGHT_ZERO_POINTS).get_shape(); + + if (zeroPointsShape != scalesShape) { return false; } } @@ -592,6 +601,7 @@ void FullyConnected::initSupportedPrimitiveDescriptors() { attrs.dynamicQuantizationGroupSize = context->getConfig().fcDynamicQuantizationGroupSize; attrs.modelType = context->getConfig().modelType; + attrs.dqScales = getDQScales(); attrs.postOps = getPostOps(fusedWith); const auto& srcTypes = getOriginalInputPrecisions(); From 8b2759fee348438c65911ab371c64ed18b55df83 Mon Sep 17 00:00:00 2001 From: "hrushit.kakadia" Date: Thu, 6 Aug 2026 18:09:29 +0530 Subject: [PATCH 3/3] fixed clang format issues and copilot comment --- .../executors/kleidiai/kleidiai_common.hpp | 282 +++++++----------- .../nodes/executors/kleidiai/kleidiai_mm.cpp | 73 +++-- .../nodes/executors/kleidiai/kleidiai_mm.hpp | 2 +- .../intel_cpu/src/nodes/fullyconnected.cpp | 18 +- 4 files changed, 160 insertions(+), 215 deletions(-) diff --git a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp index 52814f37a892..400fc4aeac0e 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp +++ b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_common.hpp @@ -1211,12 +1211,14 @@ class uKernel : public uKernelBase { size_t group_size = 0; public: - uKernel(size_t N, size_t K, MemoryPtr& lhsPackedMem, const MemoryArgs& memory) - : lhsPackedMem(lhsPackedMem) { + uKernel(size_t N, size_t K, MemoryPtr& lhsPackedMem, const MemoryArgs& memory) : lhsPackedMem(lhsPackedMem) { this->N = N; this->K = K; const auto scalesDims = memory.at(ARG_WEI | ARG_ATTR_SCALES)->getDesc().getShape().getDims(); - OPENVINO_ASSERT(scalesDims.size() > 1, "Group quantization requires scales with at least two dimensions. Got ", scalesDims.size(), "."); + OPENVINO_ASSERT(scalesDims.size() > 1, + "Group quantization requires scales with at least two dimensions. Got ", + scalesDims.size(), + "."); OPENVINO_ASSERT(scalesDims[1] != 0 && K % scalesDims[1] == 0, "Invalid scales shape for group quantization."); group_size = K / scalesDims[1]; OPENVINO_ASSERT(group_size % 32 == 0, "Group size must be a multiple of 32. Got ", group_size); @@ -1228,8 +1230,10 @@ class uKernel : public uKernelBase { OPENVINO_ASSERT(zeroPointDims == scalesDims, "Zero-point and scale tensors must have the same shape."); const auto zpPrec = rhsZeroPointsMem->getDescPtr()->getPrecision(); OPENVINO_ASSERT(zpPrec == ov::element::u4 || zpPrec == ov::element::f32, - "INT4 asymmetric group KleidiAI kernel expects u4 or f32 " - "zero-points. Got ", zpPrec, "."); + "INT4 asymmetric group KleidiAI kernel expects u4 or f32 " + "zero-points. Got ", + zpPrec, + "."); this->BLOCK_SIZE = uKernelInterface.get_m_step(); this->mr = uKernelInterface.get_mr(); this->nr = uKernelInterface.get_nr(); @@ -1238,28 +1242,20 @@ class uKernel : public uKernelBase { } size_t get_rhsPackedSize() override { - return kai_get_rhs_packed_size_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon( - N, - K, - nr, - kr, - group_size); + return kai_get_rhs_packed_size_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon(N, K, nr, kr, group_size); } void packData(bool isTransposed, - MemoryCPtr weightsMemory, - MemoryPtr biasMem, - bool hasBias, - float* rhs_scales, - MemoryPtr rhsPackedMemory) override { - OPENVINO_ASSERT(rhs_scales != nullptr, - "INT4 asymmetric group quantization requires scales."); + MemoryCPtr weightsMemory, + MemoryPtr biasMem, + bool hasBias, + float* rhs_scales, + MemoryPtr rhsPackedMemory) override { + OPENVINO_ASSERT(rhs_scales != nullptr, "INT4 asymmetric group quantization requires scales."); - OPENVINO_ASSERT(!isTransposed, - "ASYM qai4c32p path currently supports non-transposed NxK weights."); + OPENVINO_ASSERT(!isTransposed, "ASYM qai4c32p path currently supports non-transposed NxK weights."); - OPENVINO_ASSERT(rhsZeroPointsMem, - "INT4 asymmetric group quantization requires zero-points."); + OPENVINO_ASSERT(rhsZeroPointsMem, "INT4 asymmetric group quantization requires zero-points."); rhsPackedMem = rhsPackedMemory; @@ -1275,17 +1271,14 @@ class uKernel : public uKernelBase { for (size_t i = 0; i < weight_bytes; ++i) { const uint8_t b = rhs_qau4_s1s0[i]; - rhs_qau4_s0s1[i] = static_cast( - ((b & 0x0FU) << 4) | - ((b & 0xF0U) >> 4)); + rhs_qau4_s0s1[i] = static_cast(((b & 0x0FU) << 4) | ((b & 0xF0U) >> 4)); } // Read a logical U4 value from OpenVINO's packed s1s0 storage. auto read_u4_s1s0 = [](const uint8_t* data, size_t index) -> uint8_t { const uint8_t b = data[index / 2]; - return (index & 1U) ? ((b >> 4) & 0x0FU) - : (b & 0x0FU); + return (index & 1U) ? ((b >> 4) & 0x0FU) : (b & 0x0FU); }; const auto* zp_qau4_s1s0 = rhsZeroPointsMem->getDataAs(); @@ -1293,32 +1286,29 @@ class uKernel : public uKernelBase { std::vector zero_points_f32(num_zero_points); for (size_t i = 0; i < num_zero_points; ++i) { - const int32_t zp_signed = - static_cast(read_u4_s1s0(zp_qau4_s1s0, i)) - 8; + const int32_t zp_signed = static_cast(read_u4_s1s0(zp_qau4_s1s0, i)) - 8; - zero_points_f32[i] = - -static_cast(zp_signed) * rhs_scales[i]; + zero_points_f32[i] = -static_cast(zp_signed) * rhs_scales[i]; } kai_rhs_pack_nxk_qai4c32p_params params{}; params.lhs_zero_point = 1; params.rhs_zero_point = 8; - kai_run_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon( - 1, - N, - K, - nr, - kr, - sr, - group_size, - rhs_qau4_s0s1.data(), - zero_points_f32.data(), - bias_ptr, - rhs_scales, - rhsPackedMem->getData(), - 0, - ¶ms); + kai_run_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon(1, + N, + K, + nr, + kr, + sr, + group_size, + rhs_qau4_s0s1.data(), + zero_points_f32.data(), + bias_ptr, + rhs_scales, + rhsPackedMem->getData(), + 0, + ¶ms); } KernelInterface getuKernelInterface() override { return uKernelInterface; @@ -1332,13 +1322,7 @@ class uKernel : public uKernelBase { const size_t m_blocks = (m + BLOCK_SIZE - 1) / BLOCK_SIZE; packedlhs_block_in_bytes = - kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32pscalef32_f32_neon( - BLOCK_SIZE, - K, - group_size, - mr, - kr, - sr); + kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32pscalef32_f32_neon(BLOCK_SIZE, K, group_size, mr, kr, sr); return m_blocks * packedlhs_block_in_bytes; } @@ -1348,9 +1332,7 @@ class uKernel : public uKernelBase { ov::intel_cpu::Dim K, ov::intel_cpu::MemoryPtr dstMem, ov::intel_cpu::MemoryPtr srcMem) override { - const auto ukernel = - std::get( - getuKernelInterface()); + const auto ukernel = std::get(getuKernelInterface()); auto* lhs = srcMem->getDataAs(); auto* dst = dstMem->getDataAs(); @@ -1364,8 +1346,7 @@ class uKernel : public uKernelBase { const size_t m_blocks = (M + m_step - 1) / m_step; const size_t n_blocks = (N + n_step - 1) / n_step; - const size_t lhs_packed_offset = - ukernel.get_lhs_packed_offset(0, K, group_size); + const size_t lhs_packed_offset = ukernel.get_lhs_packed_offset(0, K, group_size); const size_t lhs_stride = K * sizeof(float); const size_t dst_stride_row = N * sizeof(float); @@ -1376,41 +1357,32 @@ class uKernel : public uKernelBase { cpu_parallel->parallel_for(m_blocks, [&](size_t m_blk) { const size_t m_iter = std::min(M - m_blk * m_step, m_step); - auto* lhs_packed_block = - lhs_packed + m_blk * packedlhs_block_in_bytes; - - kai_run_lhs_quant_pack_qsi8d32pscalef32_f32_neon( - m_iter, - K, - group_size, - mr, - kr, - sr, - 0, - lhs + m_blk * m_step * K, - lhs_stride, - lhs_packed_block); + auto* lhs_packed_block = lhs_packed + m_blk * packedlhs_block_in_bytes; + + kai_run_lhs_quant_pack_qsi8d32pscalef32_f32_neon(m_iter, + K, + group_size, + mr, + kr, + sr, + 0, + lhs + m_blk * m_step * K, + lhs_stride, + lhs_packed_block); cpu_parallel->parallel_for(n_blocks, [&](size_t n_blk) { const size_t n_start = n_blk * n_step; const size_t n_iter = std::min(N - n_start, n_step); - const size_t rhs_packed_offset = - ukernel.get_rhs_packed_offset(n_start, K, group_size); + const size_t rhs_packed_offset = ukernel.get_rhs_packed_offset(n_start, K, group_size); - const size_t dst_offset = - ukernel.get_dst_offset(m_blk * m_step, - n_start, - dst_stride_row); + const size_t dst_offset = ukernel.get_dst_offset(m_blk * m_step, n_start, dst_stride_row); - const auto* lhs_ptr = - static_cast(lhs_packed_block + lhs_packed_offset); + const auto* lhs_ptr = static_cast(lhs_packed_block + lhs_packed_offset); - const auto* rhs_ptr = - static_cast(rhs_packed + rhs_packed_offset); + const auto* rhs_ptr = static_cast(rhs_packed + rhs_packed_offset); - auto* dst_ptr = - dst + dst_offset / sizeof(float); + auto* dst_ptr = dst + dst_offset / sizeof(float); ukernel.run_matmul(m_iter, n_iter, @@ -1428,7 +1400,7 @@ class uKernel : public uKernelBase { } }; -template<> +template <> class uKernel : public uKernelBase { private: static constexpr kai_matmul_clamp_f32_qsi8d32p_qai4c32p_ukernel uKernelInterface{ @@ -1450,12 +1422,14 @@ class uKernel : public uKernelBase { size_t group_size = 0; public: - uKernel(size_t N, size_t K, MemoryPtr& lhsPackedMem, const MemoryArgs& memory) - : lhsPackedMem(lhsPackedMem) { + uKernel(size_t N, size_t K, MemoryPtr& lhsPackedMem, const MemoryArgs& memory) : lhsPackedMem(lhsPackedMem) { this->N = N; this->K = K; const auto scalesDims = memory.at(ARG_WEI | ARG_ATTR_SCALES)->getDesc().getShape().getDims(); - OPENVINO_ASSERT(scalesDims.size() > 1, "Group quantization requires scales with at least two dimensions. Got ", scalesDims.size(), "."); + OPENVINO_ASSERT(scalesDims.size() > 1, + "Group quantization requires scales with at least two dimensions. Got ", + scalesDims.size(), + "."); OPENVINO_ASSERT(scalesDims[1] != 0 && K % scalesDims[1] == 0, "Invalid scales shape for group quantization."); group_size = K / scalesDims[1]; OPENVINO_ASSERT(group_size % 32 == 0, "Group size must be a multiple of 32. Got ", group_size); @@ -1467,8 +1441,10 @@ class uKernel : public uKernelBase { OPENVINO_ASSERT(zeroPointDims == scalesDims, "Zero-point and scale tensors must have the same shape."); const auto zpPrec = rhsZeroPointsMem->getDescPtr()->getPrecision(); OPENVINO_ASSERT(zpPrec == ov::element::u4 || zpPrec == ov::element::f32, - "INT4 asymmetric group KleidiAI kernel expects u4 or f32 " - "zero-points. Got ", zpPrec, "."); + "INT4 asymmetric group KleidiAI kernel expects u4 or f32 " + "zero-points. Got ", + zpPrec, + "."); this->BLOCK_SIZE = uKernelInterface.get_m_step(); this->mr = uKernelInterface.get_mr(); this->nr = uKernelInterface.get_nr(); @@ -1477,28 +1453,20 @@ class uKernel : public uKernelBase { } size_t get_rhsPackedSize() override { - return kai_get_rhs_packed_size_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon( - N, - K, - nr, - kr, - group_size); + return kai_get_rhs_packed_size_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon(N, K, nr, kr, group_size); } void packData(bool isTransposed, - MemoryCPtr weightsMemory, - MemoryPtr biasMem, - bool hasBias, - float* rhs_scales, - MemoryPtr rhsPackedMemory) override { - OPENVINO_ASSERT(rhs_scales != nullptr, - "INT4 asymmetric group quantization requires scales."); + MemoryCPtr weightsMemory, + MemoryPtr biasMem, + bool hasBias, + float* rhs_scales, + MemoryPtr rhsPackedMemory) override { + OPENVINO_ASSERT(rhs_scales != nullptr, "INT4 asymmetric group quantization requires scales."); - OPENVINO_ASSERT(!isTransposed, - "ASYM qai4c32p path currently supports non-transposed NxK weights."); + OPENVINO_ASSERT(!isTransposed, "ASYM qai4c32p path currently supports non-transposed NxK weights."); - OPENVINO_ASSERT(rhsZeroPointsMem, - "INT4 asymmetric group quantization requires zero-points."); + OPENVINO_ASSERT(rhsZeroPointsMem, "INT4 asymmetric group quantization requires zero-points."); rhsPackedMem = rhsPackedMemory; @@ -1514,17 +1482,14 @@ class uKernel : public uKernelBase { for (size_t i = 0; i < weight_bytes; ++i) { const uint8_t b = rhs_qau4_s1s0[i]; - rhs_qau4_s0s1[i] = static_cast( - ((b & 0x0FU) << 4) | - ((b & 0xF0U) >> 4)); + rhs_qau4_s0s1[i] = static_cast(((b & 0x0FU) << 4) | ((b & 0xF0U) >> 4)); } // Read a logical U4 value from OpenVINO's packed s1s0 storage. auto read_u4_s1s0 = [](const uint8_t* data, size_t index) -> uint8_t { const uint8_t b = data[index / 2]; - return (index & 1U) ? ((b >> 4) & 0x0FU) - : (b & 0x0FU); + return (index & 1U) ? ((b >> 4) & 0x0FU) : (b & 0x0FU); }; const auto* zp_qau4_s1s0 = rhsZeroPointsMem->getDataAs(); @@ -1532,32 +1497,29 @@ class uKernel : public uKernelBase { std::vector zero_points_f32(num_zero_points); for (size_t i = 0; i < num_zero_points; ++i) { - const int32_t zp_signed = - static_cast(read_u4_s1s0(zp_qau4_s1s0, i)) - 8; + const int32_t zp_signed = static_cast(read_u4_s1s0(zp_qau4_s1s0, i)) - 8; - zero_points_f32[i] = - -static_cast(zp_signed) * rhs_scales[i]; + zero_points_f32[i] = -static_cast(zp_signed) * rhs_scales[i]; } kai_rhs_pack_nxk_qai4c32p_params params{}; params.lhs_zero_point = 1; params.rhs_zero_point = 8; - kai_run_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon( - 1, - N, - K, - nr, - kr, - sr, - group_size, - rhs_qau4_s0s1.data(), - zero_points_f32.data(), - bias_ptr, - rhs_scales, - rhsPackedMem->getData(), - 0, - ¶ms); + kai_run_rhs_pack_nxk_qai4c32p_qau4c32s0s1_f32_f32_f32_neon(1, + N, + K, + nr, + kr, + sr, + group_size, + rhs_qau4_s0s1.data(), + zero_points_f32.data(), + bias_ptr, + rhs_scales, + rhsPackedMem->getData(), + 0, + ¶ms); } KernelInterface getuKernelInterface() override { return uKernelInterface; @@ -1571,13 +1533,7 @@ class uKernel : public uKernelBase { const size_t m_blocks = (m + BLOCK_SIZE - 1) / BLOCK_SIZE; packedlhs_block_in_bytes = - kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32pscalef32_f32_neon( - BLOCK_SIZE, - K, - group_size, - mr, - kr, - sr); + kai_get_lhs_packed_size_lhs_quant_pack_qsi8d32pscalef32_f32_neon(BLOCK_SIZE, K, group_size, mr, kr, sr); return m_blocks * packedlhs_block_in_bytes; } @@ -1587,9 +1543,7 @@ class uKernel : public uKernelBase { ov::intel_cpu::Dim K, ov::intel_cpu::MemoryPtr dstMem, ov::intel_cpu::MemoryPtr srcMem) override { - const auto ukernel = - std::get( - getuKernelInterface()); + const auto ukernel = std::get(getuKernelInterface()); auto* lhs = srcMem->getDataAs(); auto* dst = dstMem->getDataAs(); @@ -1603,8 +1557,7 @@ class uKernel : public uKernelBase { const size_t m_blocks = (M + m_step - 1) / m_step; const size_t n_blocks = (N + n_step - 1) / n_step; - const size_t lhs_packed_offset = - ukernel.get_lhs_packed_offset(0, K, group_size); + const size_t lhs_packed_offset = ukernel.get_lhs_packed_offset(0, K, group_size); const size_t lhs_stride = K * sizeof(float); const size_t dst_stride_row = N * sizeof(float); @@ -1615,41 +1568,32 @@ class uKernel : public uKernelBase { cpu_parallel->parallel_for(m_blocks, [&](size_t m_blk) { const size_t m_iter = std::min(M - m_blk * m_step, m_step); - auto* lhs_packed_block = - lhs_packed + m_blk * packedlhs_block_in_bytes; - - kai_run_lhs_quant_pack_qsi8d32pscalef32_f32_neon( - m_iter, - K, - group_size, - mr, - kr, - sr, - 0, - lhs + m_blk * m_step * K, - lhs_stride, - lhs_packed_block); + auto* lhs_packed_block = lhs_packed + m_blk * packedlhs_block_in_bytes; + + kai_run_lhs_quant_pack_qsi8d32pscalef32_f32_neon(m_iter, + K, + group_size, + mr, + kr, + sr, + 0, + lhs + m_blk * m_step * K, + lhs_stride, + lhs_packed_block); cpu_parallel->parallel_for(n_blocks, [&](size_t n_blk) { const size_t n_start = n_blk * n_step; const size_t n_iter = std::min(N - n_start, n_step); - const size_t rhs_packed_offset = - ukernel.get_rhs_packed_offset(n_start, K, group_size); + const size_t rhs_packed_offset = ukernel.get_rhs_packed_offset(n_start, K, group_size); - const size_t dst_offset = - ukernel.get_dst_offset(m_blk * m_step, - n_start, - dst_stride_row); + const size_t dst_offset = ukernel.get_dst_offset(m_blk * m_step, n_start, dst_stride_row); - const auto* lhs_ptr = - static_cast(lhs_packed_block + lhs_packed_offset); + const auto* lhs_ptr = static_cast(lhs_packed_block + lhs_packed_offset); - const auto* rhs_ptr = - static_cast(rhs_packed + rhs_packed_offset); + const auto* rhs_ptr = static_cast(rhs_packed + rhs_packed_offset); - auto* dst_ptr = - dst + dst_offset / sizeof(float); + auto* dst_ptr = dst + dst_offset / sizeof(float); ukernel.run_matmul(m_iter, n_iter, diff --git a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp index 9bd61c72d4bc..3e2c6248f348 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp +++ b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.cpp @@ -53,11 +53,21 @@ static bool useDynamicQuantizationImpl(const FCAttrs& attrs, const MemoryDescPtr return false; } - return weightDesc->getPrecision() == element::i8 || weightDesc->getPrecision() == element::i4 || weightDesc->getPrecision() == element::u4; + return weightDesc->getPrecision() == element::i8 || weightDesc->getPrecision() == element::i4 || + weightDesc->getPrecision() == element::u4; } bool MatMulKleidiAIExecutor::supports(const FCConfig& config) { VERIFY(hasArmISASupport(ArmISA::ASIMD), UNSUPPORTED_ISA); + + // ASYM INT4 groupwise kernel is N×K only (no K×N variant); it cannot + // consume non-transposed weights, and scale/zero-point repacking for that + // layout isn't implemented. Reject so another executor is selected. + const auto& weiPrec = config.descs.at(ARG_WEI)->getPrecision(); + if (weiPrec == element::u4 && config.attrs.weightsNonTransposed) { + return false; + } + return config.descs.at(ARG_WEI)->getPrecision() == element::f32 || useDynamicQuantizationImpl(config.attrs, config.descs.at(ARG_WEI)); } @@ -72,9 +82,7 @@ bool MatMulKleidiAIExecutor::isGroupQuantizationEnabled(const MemoryArgs& memory bool MatMulKleidiAIExecutor::isAsymmetricQuantizationEnabled(const MemoryArgs& memory) { const auto zpIt = memory.find(ARG_WEI | ARG_ATTR_ZERO_POINTS); - return zpIt != memory.end() && - zpIt->second != nullptr && - !zpIt->second->getDesc().empty(); + return zpIt != memory.end() && zpIt->second != nullptr && !zpIt->second->getDesc().empty(); } MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, @@ -105,7 +113,7 @@ MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, const VectorDims wgtDims2D = reshapeDownToRank<2>(wgtDims); originalWeightsDesc = std::make_shared(originalWeightsDesc->getPrecision(), Shape{wgtDims2D}); auto dnnlSrcDesc = MemoryDescUtils::convertToDnnlMemoryDesc(originalWeightsDesc); - + kernelLookupKey = 0; const bool useDynamicQuant = useDynamicQuantizationImpl(attrs, originalWeightsDesc); @@ -118,8 +126,7 @@ MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, } else if (weightPrecision == element::i8) { kernelLookupKey |= kernelLookup::WEIGHT_INT8; } else { - OPENVINO_THROW_NOT_IMPLEMENTED("Unsupported weight format by KleidiAI executor: ", - weightPrecision); + OPENVINO_THROW_NOT_IMPLEMENTED("Unsupported weight format by KleidiAI executor: ", weightPrecision); } if (hasArmISASupport(ArmISA::I8MM)) { @@ -130,10 +137,9 @@ MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, OPENVINO_THROW_NOT_IMPLEMENTED("KleidiAI quantized kernels require Arm dotprod or i8mm support."); } - kernelLookupKey |= isGroupQuantizationEnabled(memory) ? kernelLookup::QUANT_GROUP - : kernelLookup::QUANT_CHANNEL; - kernelLookupKey |= isAsymmetricQuantizationEnabled(memory) ? kernelLookup::QUANT_ASYMMETRIC - : kernelLookup::QUANT_SYMMETRIC; + kernelLookupKey |= isGroupQuantizationEnabled(memory) ? kernelLookup::QUANT_GROUP : kernelLookup::QUANT_CHANNEL; + kernelLookupKey |= + isAsymmetricQuantizationEnabled(memory) ? kernelLookup::QUANT_ASYMMETRIC : kernelLookup::QUANT_SYMMETRIC; } const auto createPackedMemory = [&](size_t size, const element::Type& precision) { @@ -141,7 +147,7 @@ MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, rhsPackedMem = std::make_shared(context->getEngine(), desc); }; - const auto packQuantizedWeights = [&](bool isTransposed, MemoryCPtr weightsMemory) { + const auto packQuantizedWeights = [&](bool isTransposed, const MemoryCPtr& weightsMemory) { createPackedMemory(_kernel->get_rhsPackedSize(), i8); auto* rhsScales = static_cast(memory.at(ARG_WEI | ARG_ATTR_SCALES)->getData()); std::vector transposedScales; @@ -177,40 +183,45 @@ MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, } case WEIGHT_INT4 | ISA_I8MM | QUANT_GROUP | QUANT_SYMMETRIC: - _kernel = std::make_shared>( - N, K, lhsPackedMem, memory); + _kernel = std::make_shared>(N, + K, + lhsPackedMem, + memory); packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); break; case WEIGHT_INT4 | ISA_DOTPROD | QUANT_GROUP | QUANT_SYMMETRIC: - _kernel = std::make_shared>( - N, K, lhsPackedMem, memory); + _kernel = std::make_shared>(N, + K, + lhsPackedMem, + memory); packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); break; case WEIGHT_INT4 | ISA_I8MM | QUANT_GROUP | QUANT_ASYMMETRIC: - _kernel = std::make_shared>( - N, K, lhsPackedMem, memory); + _kernel = std::make_shared>(N, + K, + lhsPackedMem, + memory); packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); break; case WEIGHT_INT4 | ISA_DOTPROD | QUANT_GROUP | QUANT_ASYMMETRIC: - _kernel = std::make_shared>( - N, K, lhsPackedMem, memory); + _kernel = + std::make_shared>(N, + K, + lhsPackedMem, + memory); packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); break; case WEIGHT_INT4 | ISA_I8MM | QUANT_CHANNEL | QUANT_SYMMETRIC: - _kernel = std::make_shared>(N, - K, - lhsPackedMem); + _kernel = std::make_shared>(N, K, lhsPackedMem); packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); break; case WEIGHT_INT4 | ISA_DOTPROD | QUANT_CHANNEL | QUANT_SYMMETRIC: - _kernel = std::make_shared>(N, - K, - lhsPackedMem); + _kernel = std::make_shared>(N, K, lhsPackedMem); packQuantizedWeights(attrs.weightsNonTransposed, memory.at(ARG_WEI)); break; @@ -218,11 +229,10 @@ MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, case WEIGHT_INT8 | ISA_I8MM | QUANT_CHANNEL | QUANT_SYMMETRIC: { const bool useI8MM = (kernelLookupKey & ISA_I8MM) != 0; if (useI8MM) { - _kernel = std::make_shared>( - N, K, lhsPackedMem); + _kernel = std::make_shared>(N, K, lhsPackedMem); } else { - _kernel = std::make_shared>( - N, K, lhsPackedMem); + _kernel = + std::make_shared>(N, K, lhsPackedMem); } MemoryPtr weightsMemory = memory.at(ARG_WEI); @@ -236,8 +246,7 @@ MatMulKleidiAIExecutor::MatMulKleidiAIExecutor(const FCAttrs& attrs, } default: - OPENVINO_THROW_NOT_IMPLEMENTED("Unsupported KleidiAI kernel configuration. Lookup key: ", - kernelLookupKey); + OPENVINO_THROW_NOT_IMPLEMENTED("Unsupported KleidiAI kernel configuration. Lookup key: ", kernelLookupKey); } scratchPad = context->getScratchPad(); diff --git a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp index 05810282907f..805cfb261b31 100644 --- a/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp +++ b/src/plugins/intel_cpu/src/nodes/executors/kleidiai/kleidiai_mm.hpp @@ -33,7 +33,7 @@ class MatMulKleidiAIExecutor : public Executor { void setKaiExecutorImplAsGatherMatmul(); void set_gather_idx(const std::vector>& idxMap); - enum kernelLookup { + enum kernelLookup : std::uint16_t { WEIGHT_FP32 = 1, WEIGHT_INT8 = 1 << 2, WEIGHT_INT4 = 1 << 3, diff --git a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp index 8e65cf28ffad..a24abd1d9685 100644 --- a/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp +++ b/src/plugins/intel_cpu/src/nodes/fullyconnected.cpp @@ -199,33 +199,25 @@ bool FullyConnected::isSupportedCompressedOperation([[maybe_unused]] const std:: const auto scalesShape = op->input(WEIGHT_SCALES).get_shape(); const bool isChannelWise = shape_size(scalesShape) == OC; - const bool hasValidGroupGeometry = - IC % G == 0 && - IC / G >= 4 && - OC != 1 && - (IC / G) % 32 == 0; + const bool hasValidGroupGeometry = IC % G == 0 && IC / G >= 4 && OC != 1 && (IC / G) % 32 == 0; const bool isGroupWise = !isChannelWise && hasValidGroupGeometry; if (!isChannelWise && !isGroupWise) { return false; } - const bool hasWeightZeroPoints = - op->get_input_size() > WEIGHT_ZERO_POINTS && - op->input(WEIGHT_ZERO_POINTS).get_element_type() != ov::element::dynamic; + const bool hasWeightZeroPoints = op->get_input_size() > WEIGHT_ZERO_POINTS && + op->input(WEIGHT_ZERO_POINTS).get_element_type() != ov::element::dynamic; if (hasWeightZeroPoints) { const auto weightsType = op->input(WEIGHTS).get_element_type(); const auto zeroPointsType = op->input(WEIGHT_ZERO_POINTS).get_element_type(); // KleidiAI supports asymmetric INT4 only for group-wise u4. - if (weightsType != ov::element::u4 || - zeroPointsType != ov::element::u4 || - !isGroupWise) { + if (weightsType != ov::element::u4 || zeroPointsType != ov::element::u4 || !isGroupWise) { return false; } - const auto zeroPointsShape = - op->input(WEIGHT_ZERO_POINTS).get_shape(); + const auto zeroPointsShape = op->input(WEIGHT_ZERO_POINTS).get_shape(); if (zeroPointsShape != scalesShape) { return false;