Skip to content

[NPUW] Enable vocabulary sharing for the asymmetric vocabulary case #37054

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AsyaPronina wants to merge 10 commits into
base: master
Choose a base branch
from
Open

[NPUW] Enable vocabulary sharing for the asymmetric vocabulary case #37054

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4112c1e
[NPUW] Share 1 vocab
AsyaPronina Jul 24, 2026
a5d3c8d
U8 Vocab and ZP -> I8 Vocab and ZP
AsyaPronina Aug 5, 2026
8d41dbb
Polishing
AsyaPronina Aug 5, 2026
3559c18
Removed memory consumption by transient buffers and constants without…
AsyaPronina Aug 6, 2026
df1db01
Added I8 and U8 cases differently
AsyaPronina Aug 6, 2026
4e053db
Fixed option issue
AsyaPronina Aug 6, 2026
3e9fd41
Support of optimization. Temporarily default
AsyaPronina Aug 6, 2026
e2a0a67
Working and performant version of sharing
AsyaPronina Aug 6, 2026
f6354b4
Fixed Clang-format
AsyaPronina Aug 6, 2026
bda3966
NPUW: default-enable ASYM vocab-as-input and MatMul-first vocab optio…
esmirno Aug 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/plugins/intel_npu/src/al/include/intel_npu/config/npuw_option_defs.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,3 +90,6 @@ INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_TEXT_EMBED, bool, false, ov::intel_npu::npuw::tex
INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_KOKORO, bool, false, ov::intel_npu::npuw::kokoro, enabled, "NPUW_KOKORO", KOKORO, EXPOSED, UNCACHED, ALL)
INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_KOKORO_BLOCK_SIZE, uint64_t, 200, ov::intel_npu::npuw::kokoro, block_size, "NPUW_KOKORO_BLOCK_SIZE", KOKORO, EXPOSED, UNCACHED, ALL)
INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_KOKORO_OVERLAP_SIZE, uint64_t, 20, ov::intel_npu::npuw::kokoro, overlap_size, "NPUW_KOKORO_OVERLAP_SIZE", KOKORO, EXPOSED, UNCACHED, ALL)
INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_ASYM_VOCAB_AS_INPUT, bool, false, ov::intel_npu::npuw::llm, asym_vocab_as_input, "NPUW_LLM_ASYM_VOCAB_AS_INPUT", LLM, EXPOSED, CACHED, ALL)
INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_ASYM_I8_VOCAB_AS_INPUT, bool, false, ov::intel_npu::npuw::llm, asym_i8_vocab_as_input, "NPUW_LLM_ASYM_I8_VOCAB_AS_INPUT", LLM, EXPOSED, CACHED, ALL)
INTEL_NPU_NPUW_SIMPLE_OPT(NPUW_LLM_MATMUL_FIRST_VOCAB, bool, false, ov::intel_npu::npuw::llm, matmul_first_vocab, "NPUW_LLM_MATMUL_FIRST_VOCAB", LLM, EXPOSED, CACHED, ALL)

@dmatveev dmatveev Aug 11, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need all these options? I don't think so.

If there was found a way to share asym (u4) vocabs, let just do it by default

2 changes: 1 addition & 1 deletion src/plugins/intel_npu/src/plugin/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ cross_compiled_file(${TARGET_NAME}
ARCH AVX2 ANY
npuw/util_xarch.cpp
API npuw/util_xarch.hpp
NAME unpack_i4i8 unpack_u4i8 unpack_i4f16 unpack_i4f16_scale unpack_i4f16_z unpack_u4f16 unpack_u4f16_scale_zp unpack_u4f16_asymm_zp unpack_u4f16_z unpack_u4f32 unpack_i8f16 unpack_i8f16_scale unpack_u8f16 to_f16 copy_row_as_column transpose_i4 transpose_f16 transpose_f32 unpack_f8f16_scale
NAME unpack_i4i8 unpack_u4i8 unpack_i4f16 unpack_i4f16_scale unpack_i4f16_z unpack_u4f16 unpack_u4f16_scale_zp unpack_u4f16_asymm_zp unpack_u4f16_z unpack_u4f32 unpack_i8f16 unpack_i8f16_scale unpack_i8f16_scale_zp unpack_u8f16 to_f16 copy_row_as_column transpose_i4 transpose_f16 transpose_f32 unpack_f8f16_scale
NAMESPACE ov::npuw::util::XARCH
)

Expand Down
90 changes: 86 additions & 4 deletions src/plugins/intel_npu/src/plugin/npuw/lazy_tensor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,11 @@ Const::Const(const std::shared_ptr<ov::op::v0::Constant>& n) : m_node(n) {
m_offset = weightless_cache_attr->second.as<ov::WeightlessCacheAttribute>().bin_offset;
} else {
// See the comment in serialize() for more details
LOG_WARN("Some pattern introduced a new Constant node not present in the original weights file. We need to "
"keep it in case export occurs. This will increase memory consumption.");
LOG_WARN("Some pattern introduced a new Constant node, "
<< m_node
<< ", not present in the "
"original weights file. We need to keep it in case export occurs. This will increase "
"memory consumption.");
m_copied_if_not_in_model = ov::npuw::util::copy_tensor_from_const(m_node);
}
}
Expand All @@ -57,9 +60,9 @@ bool Const::operator==(const Const& other) const {
m_cached_ptr == other.m_cached_ptr);
}

ov::Tensor Const::eval() const {
ov::Tensor Const::eval_view() const {
if (m_node) {
return ov::npuw::util::copy_tensor_from_const(m_node);
return ov::npuw::util::tensor_from_const(m_node);
}
Comment on lines -60 to 66

@dmatveev dmatveev Aug 11, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

???


// Weightless import case. Mmmap CPU weight on demand to avoid allocating all weights at once.
Expand All @@ -83,6 +86,15 @@ ov::Tensor Const::eval() const {
return m_read_from_bin;
}

ov::Tensor Const::eval() const {
if (m_node) {
return ov::npuw::util::copy_tensor_from_const(m_node);
}
// The import branches of eval_view() don't copy - the bank takes
// ownership on its side when required
return eval_view();
}

LazyTensor::Meta Const::eval_meta() const {
if (m_node) {
return {m_node->get_shape(), m_node->get_element_type()};
Expand Down Expand Up @@ -356,6 +368,54 @@ void Gather::detach() {
w.detach();
}

std::size_t Sub128::hash() const {
std::size_t seed = std::hash<std::size_t>()(7u) + 0x9e3779b9;
seed ^= tensor.get_hash() + 0x9e3779b9;
return seed;
}

bool Sub128::operator==(const Sub128& other) const {
return tensor == other.tensor;
}

ov::Tensor Sub128::eval() const {
const auto trs = tensor.get_transformations();

ov::Tensor src;
if (trs.size() == 1 && std::holds_alternative<op::Const>(trs.front())) {
// Fused path: read straight through a zero-copy view of the source,
// skipping the intermediate copy Const::eval() would make. The view is
// only ever READ here, so this is correct for all Const flavors,
// including the deserialized ones (read-only mmap / cached bin tensor)
src = std::get<op::Const>(trs.front()).eval_view();
} else {
src = tensor.eval();
}

const auto src_type = src.get_element_type();
NPUW_ASSERT(src_type == ov::element::u8 || src_type == ov::element::i8);

ov::Tensor dst(ov::element::i8, src.get_shape());
const auto* s = static_cast<const uint8_t*>(src.data());
auto* d = dst.data<int8_t>();
for (std::size_t i = 0, n = src.get_size(); i < n; ++i) {
d[i] = static_cast<int8_t>(static_cast<int8_t>(s[i]) - 128);
}
return dst;
}
Comment on lines +381 to +405

@dmatveev dmatveev Aug 11, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Honestly this could've been a generic subtract, I see no point in leaving it 128 only


LazyTensor::Meta Sub128::eval_meta() const {
return {tensor.eval_meta().shape, ov::element::i8};
}

void Sub128::read_weight(const ov::npuw::s11n::WeightsContext& ctx) {
tensor.read_weight(ctx);
}

void Sub128::detach() {
tensor.detach();
}

} // namespace op

// Stable, permanently assigned op-type IDs.
Expand All @@ -368,6 +428,7 @@ enum class TransformType : std::uint16_t {
PERMUTE = 4,
CONVERT = 5,
GATHER = 6,
SUB128 = 7,
};

struct LazyTensorImpl {
Expand Down Expand Up @@ -430,6 +491,10 @@ ov::npuw::weights::TransformType get_transform_type(const ov::npuw::weights::op:
return ov::npuw::weights::TransformType::GATHER;
}

ov::npuw::weights::TransformType get_transform_type(const ov::npuw::weights::op::Sub128&) {
return ov::npuw::weights::TransformType::SUB128;
}

} // namespace

namespace ov {
Expand Down Expand Up @@ -500,6 +565,10 @@ void Gather::serialize(ov::npuw::orc::Stream& stream) {
}
}

void Sub128::serialize(ov::npuw::orc::Stream& stream) {
stream & tensor;
}

} // namespace op

void LazyTensorImpl::serialize(ov::npuw::orc::Stream& stream) {
Expand Down Expand Up @@ -537,6 +606,9 @@ void LazyTensorImpl::serialize(ov::npuw::orc::Stream& stream) {
case TransformType::GATHER:
m_transform.emplace<op::Gather>(ov::npuw::orc::load_versioned_payload<op::Gather>(section));
break;
case TransformType::SUB128:
m_transform.emplace<op::Sub128>(ov::npuw::orc::load_versioned_payload<op::Sub128>(section));
Comment on lines +609 to +610
break;
default:
OPENVINO_THROW("ORC LazyTensor: unknown op_type ", section.type, " — please upgrade NPUW");
break;
Expand Down Expand Up @@ -649,6 +721,10 @@ void LazyTensorImpl::get_transformations(std::vector<LazyTensor::Transform>& vec
auto next_tr = op.w.get_transformations();
vec.insert(vec.end(), next_tr.begin(), next_tr.end());
},
[&vec](const op::Sub128& op) {
auto next_tr = op.tensor.get_transformations();
vec.insert(vec.end(), next_tr.begin(), next_tr.end());
},
},
m_transform);
}
Expand Down Expand Up @@ -688,6 +764,12 @@ LazyTensor LazyTensor::convert(const ov::element::Type& type) {
return new_lt;
}

LazyTensor LazyTensor::sub128() {
LazyTensor new_lt;
new_lt.m_impl = std::make_shared<LazyTensorImpl>(op::Sub128(*this));
return new_lt;
}

bool LazyTensor::operator==(const LazyTensor& other) const {
if (!m_impl && !other.m_impl) {
return true;
Expand Down
31 changes: 30 additions & 1 deletion src/plugins/intel_npu/src/plugin/npuw/lazy_tensor.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class Unpack;
class Permute;
class Convert;
class Gather;
class Sub128;
} // namespace op

class LazyTensor {
Expand All @@ -41,7 +42,8 @@ class LazyTensor {
ov::npuw::weights::op::Unpack,
ov::npuw::weights::op::Permute,
ov::npuw::weights::op::Convert,
ov::npuw::weights::op::Gather>;
ov::npuw::weights::op::Gather,
ov::npuw::weights::op::Sub128>;

LazyTensor() = default;
LazyTensor(const std::shared_ptr<ov::op::v0::Constant>& const_ptr);
Expand All @@ -58,6 +60,7 @@ class LazyTensor {

LazyTensor permute(const std::vector<std::size_t>& axes);
LazyTensor convert(const ov::element::Type& type);
LazyTensor sub128(); // byte-wise -128 zero-point shift (u8/i8 -> symmetric i8)

bool operator==(const LazyTensor& other) const;
bool operator!=(const LazyTensor& other) const;
Expand Down Expand Up @@ -97,6 +100,7 @@ class Const {
std::size_t hash() const;
bool operator==(const Const& other) const;
ov::Tensor eval() const;
ov::Tensor eval_view() const;
LazyTensor::Meta eval_meta() const;
void read_weight(const ov::npuw::s11n::WeightsContext& ctx);
void detach();
Expand Down Expand Up @@ -240,6 +244,31 @@ class Gather {
ov::element::Type dst_type;
ov::Shape dst_shape;
};

class Sub128 {
friend struct ov::npuw::weights::LazyTensorImpl;

public:
static constexpr std::uint16_t kVersion = 0u;

// rt_info marker: a graph-level pass puts this key on a Constant to request
// the -128 shift to be applied when the weight is lifted into a closure
static constexpr const char* rt_key = "npuw::sub128";

Sub128() = default;
explicit Sub128(const LazyTensor& _tensor) : tensor(_tensor) {}

std::size_t hash() const;
bool operator==(const Sub128& other) const;
ov::Tensor eval() const;
LazyTensor::Meta eval_meta() const;
void read_weight(const ov::npuw::s11n::WeightsContext& ctx);
void detach();
void serialize(ov::npuw::orc::Stream& stream);

private:
LazyTensor tensor;
};
} // namespace op

} // namespace weights
Expand Down
Loading