From 3c49e4eef9005bb7c5bac3668c0568d24d112c74 Mon Sep 17 00:00:00 2001 From: Siming Chen <867965859@qq.com> Date: Thu, 13 Aug 2026 11:09:22 +0800 Subject: [PATCH 1/3] [QNN:Bugfix] Protect index tensors during NPU quantization --- .../llm/export/utils/omni_quantizer.py | 104 +++++++++++++---- .../llm/export/utils/smooth_quantizer.py | 105 ++++++++++++++---- 2 files changed, 163 insertions(+), 46 deletions(-) diff --git a/transformers/llm/export/utils/omni_quantizer.py b/transformers/llm/export/utils/omni_quantizer.py index b1f6e78f4f..6354f0aa75 100644 --- a/transformers/llm/export/utils/omni_quantizer.py +++ b/transformers/llm/export/utils/omni_quantizer.py @@ -812,13 +812,68 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): DATA_SELECT_OPS = ['Gather', 'GatherV2', 'GatherND'] + # Keep index/shape tensors out of activation quantization. In + # particular, position_ids is INT32 and must not become the input of a + # QNN Reshape whose output is UFIXED_POINT_16. + protected = set() + integer_types = { + 'DT_INT8', 'DT_INT16', 'DT_INT32', 'DT_INT64', + 'DT_UINT8', 'DT_UINT16', 'DT_UINT32', 'DT_UINT64', + } + for op in mnn_ops: + op_type = op.get('type', '') + inputs = op.get('inputIndexes', []) + outputs = op.get('outputIndexes', []) + main = op.get('main') or {} + if op_type == 'Input' and main.get('dtype') in integer_types: + protected.update(outputs) + elif op_type == 'Const' and main.get('dataType') in integer_types: + protected.update(outputs) + elif op_type in {'Shape', 'Rank', 'Size'}: + protected.update(outputs) + if op_type == 'Reshape' and len(inputs) > 1: + protected.update(inputs[1:]) + elif op_type in DATA_SELECT_OPS and len(inputs) > 1: + protected.update(inputs[1:]) + + # Propagate the protected marker through shape/index-only arithmetic. + changed_protected = True + while changed_protected: + changed_protected = False + for op in mnn_ops: + op_type = op.get('type', '') + inputs = op.get('inputIndexes', []) + outputs = op.get('outputIndexes', []) + if not inputs or not outputs: + continue + if op_type in PASS_THROUGH_OPS: + # For Reshape/Slice/etc. only the data input determines + # the output type; shape/axis inputs must not contaminate + # the feature-map output. + mark = inputs[0] in protected + elif op_type in {'BinaryOp', 'UnaryOp'}: + mark = any(index in protected for index in inputs) + else: + mark = False + if mark: + for index in outputs: + if index not in protected: + protected.add(index) + changed_protected = True + + for index in protected: + quant_info_dict.pop(index, None) + print("Start propagating quantization parameters...") changed = True pass_round = 0 + max_passes = len(mnn_ops) + 1 while changed: changed = False pass_round += 1 + if pass_round > max_passes: + raise RuntimeError("Quantization parameter propagation did not converge") update_count = 0 for op in mnn_ops: @@ -832,15 +887,16 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): if op_type in PASS_THROUGH_OPS: source_info = None for inp_idx in inputs: - if inp_idx in quant_info_dict: + if inp_idx in quant_info_dict and inp_idx not in protected: source_info = quant_info_dict[inp_idx] break if source_info: for out_idx in outputs: - if out_idx not in quant_info_dict: - quant_info_dict[out_idx] = copy.deepcopy(source_info) - quant_info_dict[out_idx]['index'] = out_idx # 修正 index + if out_idx not in protected and out_idx not in quant_info_dict: + new_info = copy.deepcopy(source_info) + new_info['index'] = out_idx + quant_info_dict[out_idx] = new_info changed = True update_count += 1 @@ -852,9 +908,10 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): if target_info: for inp_idx in inputs: - if inp_idx not in quant_info_dict: - quant_info_dict[inp_idx] = copy.deepcopy(target_info) - quant_info_dict[inp_idx]['index'] = inp_idx + if inp_idx not in quant_info_dict and inp_idx not in protected: + new_info = copy.deepcopy(target_info) + new_info['index'] = inp_idx + quant_info_dict[inp_idx] = new_info changed = True update_count += 1 @@ -863,26 +920,25 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): out_idx = outputs[0] # Forward: Data -> Output - if data_idx in quant_info_dict and out_idx not in quant_info_dict: - quant_info_dict[out_idx] = copy.deepcopy(quant_info_dict[data_idx]) - quant_info_dict[out_idx]['index'] = out_idx - changed = True - update_count += 1 - - # Backward: Output -> Data - if out_idx in quant_info_dict and data_idx not in quant_info_dict: - quant_info_dict[data_idx] = copy.deepcopy(quant_info_dict[out_idx]) - quant_info_dict[data_idx]['index'] = data_idx - changed = True - update_count += 1 + if data_idx in quant_info_dict and out_idx not in protected: + source_info = quant_info_dict[data_idx] + output_info = quant_info_dict.get(out_idx) + if output_info is None or output_info.get('quantInfo') != source_info.get('quantInfo'): + quant_info_dict[out_idx] = copy.deepcopy(source_info) + quant_info_dict[out_idx]['index'] = out_idx + changed = True + update_count += 1 + + # Do not propagate Gather output parameters back to the + # data input: QNN requires Gather data/output parameters to + # match, while the indices input is always unquantized. elif op_type == 'BinaryOp': out_idx = outputs[0] - - if out_idx in quant_info_dict: + if out_idx in quant_info_dict and out_idx not in protected: target_info = quant_info_dict[out_idx] for inp_idx in inputs: - if inp_idx not in quant_info_dict: + if inp_idx not in quant_info_dict and inp_idx not in protected: quant_info_dict[inp_idx] = copy.deepcopy(target_info) quant_info_dict[inp_idx]['index'] = inp_idx changed = True @@ -891,8 +947,10 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): else: scales = [] valid_inputs = [] + if out_idx in protected: + continue for inp_idx in inputs: - if inp_idx in quant_info_dict: + if inp_idx in quant_info_dict and inp_idx not in protected: scales.append(quant_info_dict[inp_idx]['quantInfo']['scale']) valid_inputs.append(inp_idx) diff --git a/transformers/llm/export/utils/smooth_quantizer.py b/transformers/llm/export/utils/smooth_quantizer.py index 27f747f4d0..9528cf09d8 100644 --- a/transformers/llm/export/utils/smooth_quantizer.py +++ b/transformers/llm/export/utils/smooth_quantizer.py @@ -621,14 +621,70 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): # Gather: Output Scale == Input[0] (Data) Scale. (Input[1] 是 indices，不需要) DATA_SELECT_OPS = ['Gather', 'GatherV2', 'GatherND'] + # Keep index/shape tensors out of activation quantization. In + # particular, position_ids is INT32 and must not become the input of a + # QNN Reshape whose output is UFIXED_POINT_16. + protected = set() + integer_types = { + 'DT_INT8', 'DT_INT16', 'DT_INT32', 'DT_INT64', + 'DT_UINT8', 'DT_UINT16', 'DT_UINT32', 'DT_UINT64', + } + for op in mnn_ops: + op_type = op.get('type', '') + inputs = op.get('inputIndexes', []) + outputs = op.get('outputIndexes', []) + main = op.get('main') or {} + if op_type == 'Input' and main.get('dtype') in integer_types: + protected.update(outputs) + elif op_type == 'Const' and main.get('dataType') in integer_types: + protected.update(outputs) + elif op_type in {'Shape', 'Rank', 'Size'}: + protected.update(outputs) + if op_type == 'Reshape' and len(inputs) > 1: + protected.update(inputs[1:]) + elif op_type in DATA_SELECT_OPS and len(inputs) > 1: + protected.update(inputs[1:]) + + # Propagate protection only along the tensor that carries shape/index + # values. For pass-through ops, input 0 is feature data; auxiliary + # shape/axis inputs must not make the feature output an integer tensor. + changed_protected = True + while changed_protected: + changed_protected = False + for op in mnn_ops: + op_type = op.get('type', '') + inputs = op.get('inputIndexes', []) + outputs = op.get('outputIndexes', []) + if not inputs or not outputs: + continue + if op_type in PASS_THROUGH_OPS: + # Reshape/Slice/etc. derive the output dtype from their + # feature-data input, never from shape/axis inputs. + mark = inputs[0] in protected + elif op_type in {'BinaryOp', 'UnaryOp'}: + mark = any(index in protected for index in inputs) + else: + mark = False + if mark: + for index in outputs: + if index not in protected: + protected.add(index) + changed_protected = True + + for index in protected: + quant_info_dict.pop(index, None) + print("Start propagating quantization parameters...") changed = True pass_round = 0 # 不动点迭代：只要这轮循环有更新，就继续跑下一轮 + max_passes = len(mnn_ops) + 1 while changed: changed = False pass_round += 1 + if pass_round > max_passes: + raise RuntimeError("Quantization parameter propagation did not converge") update_count = 0 for op in mnn_ops: @@ -648,15 +704,16 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): # 通常取第一个有参数的 input 作为源 source_info = None for inp_idx in inputs: - if inp_idx in quant_info_dict: + if inp_idx in quant_info_dict and inp_idx not in protected: source_info = quant_info_dict[inp_idx] break if source_info: for out_idx in outputs: - if out_idx not in quant_info_dict: - quant_info_dict[out_idx] = copy.deepcopy(source_info) - quant_info_dict[out_idx]['index'] = out_idx # 修正 index + if out_idx not in protected and out_idx not in quant_info_dict: + new_info = copy.deepcopy(source_info) + new_info['index'] = out_idx + quant_info_dict[out_idx] = new_info changed = True update_count += 1 @@ -670,9 +727,10 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): if target_info: for inp_idx in inputs: - if inp_idx not in quant_info_dict: - quant_info_dict[inp_idx] = copy.deepcopy(target_info) - quant_info_dict[inp_idx]['index'] = inp_idx + if inp_idx not in quant_info_dict and inp_idx not in protected: + new_info = copy.deepcopy(target_info) + new_info['index'] = inp_idx + quant_info_dict[inp_idx] = new_info changed = True update_count += 1 @@ -684,18 +742,18 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): out_idx = outputs[0] # Forward: Data -> Output - if data_idx in quant_info_dict and out_idx not in quant_info_dict: - quant_info_dict[out_idx] = copy.deepcopy(quant_info_dict[data_idx]) - quant_info_dict[out_idx]['index'] = out_idx - changed = True - update_count += 1 - - # Backward: Output -> Data - if out_idx in quant_info_dict and data_idx not in quant_info_dict: - quant_info_dict[data_idx] = copy.deepcopy(quant_info_dict[out_idx]) - quant_info_dict[data_idx]['index'] = data_idx - changed = True - update_count += 1 + if data_idx in quant_info_dict and out_idx not in protected: + source_info = quant_info_dict[data_idx] + output_info = quant_info_dict.get(out_idx) + if output_info is None or output_info.get('quantInfo') != source_info.get('quantInfo'): + quant_info_dict[out_idx] = copy.deepcopy(source_info) + quant_info_dict[out_idx]['index'] = out_idx + changed = True + update_count += 1 + + # Do not propagate Gather output parameters back to the + # data input: QNN requires Gather data/output parameters to + # match, while the indices input is always unquantized. # ----------------------------------------------- # 策略 3: BinaryOp (Add/Mul) - 谨慎处理 @@ -703,16 +761,15 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): # ----------------------------------------------- elif op_type == 'BinaryOp': out_idx = outputs[0] - # Backward: # 如果 Add 的输出已知（通常是因为连着下一个 Linear/Norm 的输入）， # 我们可以尝试推导输入的 Scale。 # 注意：对于 Add，如果 Input A 和 Input B 的范围差异巨大，直接回传可能有风险。 # 但在 Transformer 残差结构中，通常 Input 和 Output 的 Scale 是同数量级的。 - if out_idx in quant_info_dict: + if out_idx in quant_info_dict and out_idx not in protected: target_info = quant_info_dict[out_idx] for inp_idx in inputs: - if inp_idx not in quant_info_dict: + if inp_idx not in quant_info_dict and inp_idx not in protected: quant_info_dict[inp_idx] = copy.deepcopy(target_info) quant_info_dict[inp_idx]['index'] = inp_idx changed = True @@ -722,10 +779,12 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): # 如果所有输入都有 Scale，取 Scale 最大的那个传给输出 # (保守策略，避免截断) else: + if out_idx in protected: + continue scales = [] valid_inputs = [] for inp_idx in inputs: - if inp_idx in quant_info_dict: + if inp_idx in quant_info_dict and inp_idx not in protected: scales.append(quant_info_dict[inp_idx]['quantInfo']['scale']) valid_inputs.append(inp_idx) From 454c38053e515f722955517fc0852b650b965f33 Mon Sep 17 00:00:00 2001 From: Siming Chen <867965859@qq.com> Date: Thu, 13 Aug 2026 11:10:10 +0800 Subject: [PATCH 2/3] [QNN:Bugfix] Preserve calibrated MUL activation ranges --- transformers/llm/export/utils/omni_quantizer.py | 9 +++++++++ transformers/llm/export/utils/smooth_quantizer.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/transformers/llm/export/utils/omni_quantizer.py b/transformers/llm/export/utils/omni_quantizer.py index 6354f0aa75..d34d60a959 100644 --- a/transformers/llm/export/utils/omni_quantizer.py +++ b/transformers/llm/export/utils/omni_quantizer.py @@ -719,6 +719,9 @@ def _collect_feature_map_optimized(self): for idx in tqdm(range(len(self.modules)), desc="Collecting Feature Map Info"): block = self.modules[idx] + # Calibration math is implemented in float32; normalize BF16 + # checkpoints before collecting activation ranges. + block.float() self.to_device(block, self.best_device) target_ops = SmoothQuantizer.get_all_leaf_modules(block) @@ -935,6 +938,12 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): elif op_type == 'BinaryOp': out_idx = outputs[0] + binary_type = (op.get('main') or {}).get('opType', '') + # Multiplication changes the value range. Neither operand + # nor the product can safely inherit the other's scale; + # retain only independently calibrated parameters. + if binary_type == 'MUL': + continue if out_idx in quant_info_dict and out_idx not in protected: target_info = quant_info_dict[out_idx] for inp_idx in inputs: diff --git a/transformers/llm/export/utils/smooth_quantizer.py b/transformers/llm/export/utils/smooth_quantizer.py index 9528cf09d8..b71d4f34b7 100644 --- a/transformers/llm/export/utils/smooth_quantizer.py +++ b/transformers/llm/export/utils/smooth_quantizer.py @@ -761,6 +761,12 @@ def _propagate_quant_info(self, mnn_ops, quant_info_dict): # ----------------------------------------------- elif op_type == 'BinaryOp': out_idx = outputs[0] + binary_type = (op.get('main') or {}).get('opType', '') + # Multiplication changes the value range. Neither operand + # nor the product can safely inherit the other's scale; + # retain only independently calibrated parameters. + if binary_type == 'MUL': + continue # Backward: # 如果 Add 的输出已知（通常是因为连着下一个 Linear/Norm 的输入）， # 我们可以尝试推导输入的 Scale。 From 258edd3bcd3f62cada60aa287767ae067e2ea610 Mon Sep 17 00:00:00 2001 From: Siming Chen <867965859@qq.com> Date: Thu, 13 Aug 2026 15:11:37 +0800 Subject: [PATCH 3/3] [QNN:Bugfix] Fix Qwen3-VL offline graph inputs --- transformers/llm/engine/src/omni.cpp | 47 +++++++++++++++++-- transformers/llm/export/llmexport.py | 6 ++- .../llm/export/npu/generate_llm_qnn.py | 45 +++++++++--------- 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/transformers/llm/engine/src/omni.cpp b/transformers/llm/engine/src/omni.cpp index 2074d7bbd0..7a65ad77de 100644 --- a/transformers/llm/engine/src/omni.cpp +++ b/transformers/llm/engine/src/omni.cpp @@ -825,6 +825,15 @@ std::vector Omni::qwen2VisionProcess(VARP image) { MNN::Express::Variable::save({patches, position_ids, attention_mask}, "input.mnn"); #endif auto outputs = mVisionModule->onForward(moduleInputs); + if (outputs.empty() || outputs[0] == nullptr || outputs[0]->getInfo() == nullptr) { + MNN_ERROR("Qwen VL vision module returned no valid image embedding.\n"); + return {}; + } + if (isQwen3VL && outputs.size() != 2) { + MNN_ERROR("Qwen3-VL vision module requires image_embeds and deepstack_feature, but got %zu outputs.\n", + outputs.size()); + return {}; + } auto imageEmbedding = outputs[0]; if (outputs.size() == 2) { mDeepStackEmbeddings.push_back(outputs[1]); @@ -1766,8 +1775,12 @@ VARP Omni::gen_position_ids(int seq_len) { }; for (int i = 0; i < seq_len; i++) { for (int axis = 0; axis < axes; axis++) { - int offset = (hunyuan && axis > 0) ? 0 : mContext->all_seq_len; - ptr[i + seq_len * axis] = axisValue(axis, i) + offset; + if (hunyuan) { + int offset = axis > 0 ? 0 : mContext->all_seq_len; + ptr[i + seq_len * axis] = axisValue(axis, i) + offset; + } else { + ptr[i + seq_len * axis] = axisValue(axis, mContext->all_seq_len + i); + } } } if (mTalker) { @@ -1785,7 +1798,35 @@ VARP Omni::gen_position_ids(int seq_len) { std::vector Omni::forwardRaw(Express::VARP hiddenState, Express::VARP mask, Express::VARP inputPos, Express::VARPS extraArgs) { MNN::Express::ExecutorScope s(mExecutor); - extraArgs.insert(extraArgs.end(), mExtraArgs.begin(), mExtraArgs.end()); + if (mConfig->has_deepstack() && mExtraArgs.size() == 1) { + auto deepstack = mExtraArgs[0]; + auto deepstackInfo = deepstack->getInfo(); + auto hiddenInfo = hiddenState->getInfo(); + if (deepstackInfo != nullptr && hiddenInfo != nullptr && deepstackInfo->dim.size() == 3) { + const int targetLength = hiddenInfo->dim[mSeqLenIndex]; + const int sourceLength = deepstackInfo->dim[1]; + const int hiddenSize = deepstackInfo->dim[2]; + const int sourceOffset = mContext->gen_seq_len > 0 ? sourceLength : mContext->all_seq_len; + if (sourceOffset != 0 || sourceLength != targetLength) { + auto alignedDeepstack = Express::_Input({deepstackInfo->dim[0], targetLength, hiddenSize}, NCHW); + auto dst = alignedDeepstack->writeMap(); + const auto src = deepstack->readMap(); + ::memset(dst, 0, alignedDeepstack->getInfo()->size * sizeof(float)); + const int copyLength = std::max(0, std::min(targetLength, sourceLength - sourceOffset)); + if (src != nullptr && copyLength > 0) { + for (int i = 0; i < deepstackInfo->dim[0]; ++i) { + ::memcpy(dst + i * targetLength * hiddenSize, + src + (i * sourceLength + sourceOffset) * hiddenSize, + copyLength * hiddenSize * sizeof(float)); + } + } + deepstack = alignedDeepstack; + } + } + extraArgs.emplace_back(deepstack); + } else { + extraArgs.insert(extraArgs.end(), mExtraArgs.begin(), mExtraArgs.end()); + } if (mIsEmbedding) { std::vector inputs{hiddenState, mask, inputPos}; if (!extraArgs.empty()) { diff --git a/transformers/llm/export/llmexport.py b/transformers/llm/export/llmexport.py index 7293adcc34..6a4a8175d5 100644 --- a/transformers/llm/export/llmexport.py +++ b/transformers/llm/export/llmexport.py @@ -396,7 +396,11 @@ def export_config(self, mnn_config = False): config['dit_solver'] = 1 if self.model_type == "gemma3": config.update({'precision': "normal"}) - if (hasattr(self, 'visual') and self.visual is not None) or (hasattr(self, 'visual') and self.audio is not None): + is_visual = hasattr(self, 'visual') and self.visual is not None + is_audio = hasattr(self, 'audio') and self.audio is not None + if is_visual or is_audio: + config['is_visual'] = is_visual + config['is_audio'] = is_audio config['mllm'] = { 'backend_type': "cpu", "thread_num": 4, diff --git a/transformers/llm/export/npu/generate_llm_qnn.py b/transformers/llm/export/npu/generate_llm_qnn.py index 9b441f4723..3d929d4fb2 100644 --- a/transformers/llm/export/npu/generate_llm_qnn.py +++ b/transformers/llm/export/npu/generate_llm_qnn.py @@ -46,7 +46,7 @@ def is_embedding_model(config_data, model_dir): return True return False -def makeIOJson(args, seq_len, hidden_size, mask_type, is_embedding=False): +def makeIOJson(args, seq_len, hidden_size, mask_type, is_embedding=False, is_mrope=False, has_deepstack=False): def model_inputs(current_seq_len, logits_index=None): inputs = [ { @@ -104,37 +104,34 @@ def model_inputs(current_seq_len, logits_index=None): inp["shape"] = [2, 1, 1, 1, 3] if inp["name"] == "position_ids": inp["shape"] = [3, 1] - if not is_embedding and "Qwen" in args.model and "VL" in args.model: + if not is_embedding and is_mrope: cfg = config["configs"] inputs = cfg[0]["inputs"] for inp in inputs: if inp["name"] == "position_ids": inp["shape"] = [3, seq_len] - - new_input = { - "name": "deepstack_embeds", - "shape": [3, 1, 1] - } - inputs.append(new_input) - inputs = cfg[1]["inputs"] for inp in inputs: if inp["name"] == "position_ids": inp["shape"] = [3, 1] - new_input = { + if not is_embedding and has_deepstack: + config["configs"][0]["inputs"].append({ "name": "deepstack_embeds", - "shape": [3, 1, 1] - } - inputs.append(new_input) + "shape": [3, seq_len, hidden_size] + }) + config["configs"][1]["inputs"].append({ + "name": "deepstack_embeds", + "shape": [3, 1, hidden_size] + }) cache = os.path.join(os.getcwd(), args.cache_path) with open(os.path.join(cache, 'input.json'), 'w') as f: f.write(json.dumps(config, indent=4)) -def makeVLIOJson(args, image_sizes): +def makeVLIOJson(args, image_sizes, model_type): configs = [] for w, h in image_sizes: - if "Qwen2.5" in args.model and "VL" in args.model: + if model_type == "qwen2_5_vl": align_size = 28 grid_h = (round(h / align_size) * align_size) // 14 grid_w = (round(w / align_size) * align_size) // 14 @@ -148,7 +145,7 @@ def makeVLIOJson(args, image_sizes): ], "outputs": ["image_embeds"] } - elif "Qwen3" in args.model or "Qwen3.5" in args.model: + elif model_type in {"qwen3_vl", "qwen3_vl_moe", "qwen3_5_vl"}: align_size = 32 grid_h = (round(h / align_size) * align_size) // 16 grid_w = (round(w / align_size) * align_size) // 16 @@ -161,9 +158,9 @@ def makeVLIOJson(args, image_sizes): {"name": "idx_tensor", "shape": [4, seq_len]}, {"name": "weight_tensor", "shape": [4, seq_len]} ], - "outputs": ["image_embeds"] + "outputs": ["image_embeds", "deepstack_feature"] } - elif "FastVLM" in args.model: + elif model_type in {"fastvlm", "llava_qwen2"}: config = { "inputs": [ {"name": "input_images", "shape": [1, 3, h, w]} @@ -171,7 +168,7 @@ def makeVLIOJson(args, image_sizes): "outputs": ["image_embeds"] } else: - raise ValueError(f"Unsupported visual model: {args.model}") + raise ValueError(f"Unsupported visual model type: {model_type}") configs.append(config) full_config = {"configs": configs} @@ -347,10 +344,12 @@ def convert_visual(args): if not image_sizes: print("No valid image sizes provided.") sys.exit(1) - if "FastVLM" in args.model: + with open(os.path.join(args.model, "llm_config.json"), "r", encoding="utf-8") as f: + model_type = json.load(f)["model_type"] + if model_type in {"fastvlm", "llava_qwen2"}: convert_fastvlm(args, image_sizes) else: - makeVLIOJson(args, image_sizes) + makeVLIOJson(args, image_sizes, model_type) inputjson = os.path.join(cache, 'input.json') ids = list(range(len(image_sizes))) convert_qnn(args, 'visual.mnn', inputjson, external_file, ids) @@ -371,11 +370,13 @@ def convert_llm(args): if "attention_mask" in config_data: mask_type = config_data["attention_mask"] is_embedding = is_embedding_model(config_data, model_dir) + is_mrope = config_data.get("is_mrope", False) + has_deepstack = config_data.get("has_deepstack", False) ids = [0, 1] model_name = 'embedding.mnn' if is_embedding else 'llm.mnn' external_file = os.path.join(model_dir, model_name + '.weight') - makeIOJson(args, args.chunk_size, hidden_size, mask_type, is_embedding) + makeIOJson(args, args.chunk_size, hidden_size, mask_type, is_embedding, is_mrope, has_deepstack) inputjson = os.path.join(cache, 'input.json') convert_qnn(args, model_name, inputjson, external_file, ids)