diff --git a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst index 4b77d7b2d08b4d..818b1550ef4e06 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst @@ -3,7 +3,7 @@ Most Efficient Large Language Models for AI PC This page is regularly updated to help you identify the best-performing LLMs on the Intel® Core™ Ultra processor family and AI PCs. -The current data is as of OpenVINO 2026.2, 28 May 2026. +The current data is as of OpenVINO 2026.3, 13 August 2026. The tables below list the key performance indicators for inference on built-in GPUs. diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models_5-330.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_5-330.csv index c71421f0473159..121b59f12ce17e 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/llm_models_5-330.csv +++ b/docs/sphinx_setup/_static/benchmarks_files/llm_models_5-330.csv @@ -1,392 +1,393 @@ Topology,Precision,Input Size,1st latency (ms),2nd latency (ms),2nd token per sec -minicpm4-0.5b,INT4-MIXED,32,22,7.3,136.99 -minicpm4-0.5b,INT4-MIXED,32,21.9,7.5,133.33 -minicpm4-0.5b,INT4-MIXED,32,23.4,7.7,129.87 -gemma-3-270m,INT4-MIXED,1024,74.2,8.2,121.95 -minicpm4-0.5b,INT4-MIXED,1024,111.5,8.4,119.05 -gemma-3-270m,INT4-MIXED,32,23.7,8.5,117.65 -minicpm4-0.5b,INT4-MIXED,1024,116.9,8.6,116.28 -minicpm4-0.5b,INT4-MIXED,1024,142.4,8.8,113.64 -gemma-3-270m,INT8-CW,1024,71.1,9,111.11 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,22.9,9,111.11 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,24.1,9.1,109.89 -gemma-3-270m,INT8-CW,32,23.5,9.2,108.70 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,24.1,9.3,107.53 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,116.7,9.8,102.04 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,118.7,10,100.00 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,147.3,10.1,99.01 -minicpm4-0.5b,INT8-CW,32,23.5,10.4,96.15 -whisper-small,INT4-MIXED,prompt0,140.5,10.5,95.24 -whisper-small,INT4-MIXED,prompt1,196.1,11,90.91 -whisper-small,INT4-MIXED,prompt1,204,11.4,87.72 -minicpm4-0.5b,INT8-CW,1024,129.2,11.5,86.96 -whisper-small,INT4-MIXED,prompt0,143.7,11.5,86.96 -whisper-small,INT4-MIXED,prompt0,157.4,11.6,86.21 -whisper-small,INT8-CW,prompt1,209.3,11.6,86.21 -whisper-small,INT4-MIXED,prompt1,223.1,11.7,85.47 -whisper-small,INT8-CW,prompt0,170,11.8,84.75 -qwen2.5-coder-0.5b-instruct,INT8-CW,32,24.9,12.1,82.64 -gemma-3-270m,FP16,32,23.6,12.3,81.30 -nanollava,INT4-MIXED,760,250.2,12.4,80.65 -whisper-large-v3-turbo,INT8-CW,prompt1,484.9,12.4,80.65 -whisper-large-v3-turbo,INT4-MIXED,prompt1,508.3,12.7,78.74 -whisper-large-v3-turbo,INT8-CW,prompt0,421.2,12.7,78.74 -whisper-large-v3-turbo,INT4-MIXED,prompt0,452.6,12.8,78.13 -gemma-3-270m,FP16,1024,79.8,12.9,77.52 -qwen2.5-coder-0.5b-instruct,INT8-CW,1024,135.3,12.9,77.52 -tiny-llama-1.1b-chat,INT4-MIXED,32,30,13.2,75.76 -nanollava,INT8-CW,760,239.5,13.7,72.99 -qwen3.5-0.8b,INT4-MIXED,85,92.7,14.3,69.93 -nanollava,INT4-MIXED,1752,485.2,14.4,69.44 -qwen3.5-0.8b,INT4-MIXED,1024,380.5,14.4,69.44 -qwen3.5-0.8b,INT4-MIXED,85,92.4,14.4,69.44 -whisper-small,FP16,prompt0,143.4,14.6,68.49 -whisper-small,FP16,prompt1,204.3,14.7,68.03 -qwen3.5-0.8b,INT4-MIXED,85,97.4,14.8,67.57 -qwen3.5-0.8b,INT4-MIXED,1024,384.7,14.8,67.57 -qwen3.5-0.8b,INT4-MIXED,1024,434.7,15.1,66.23 -tiny-llama-1.1b-chat,INT4-MIXED,1024,240.6,15.1,66.23 -nanollava,INT8-CW,1752,458.6,15.8,63.29 -tiny-llama-1.1b-chat,INT4-MIXED,32,34.3,16,62.50 -llama-3.2-1b-instruct,INT4-MIXED,32,33.1,16.4,60.98 -whisper-large-v3-turbo,FP16,prompt0,454.7,16.4,60.98 -whisper-large-v3-turbo,FP16,prompt1,521.8,16.4,60.98 -llama-3.2-1b-instruct,INT4-MIXED,32,33.8,16.6,60.24 -llama-3.2-1b-instruct,INT4-MIXED,1024,226.8,17.7,56.50 -tiny-llama-1.1b-chat,INT4-MIXED,1024,320.6,17.9,55.87 -gemma-3-1b-it,INT4-MIXED,32,33.3,18,55.56 -llama-3.2-1b-instruct,INT4-MIXED,1024,271,18,55.56 -qwen3.5-0.8b,INT8-CW,85,98.6,18,55.56 -qwen3.5-0.8b,INT8-CW,1024,398.2,18.6,53.76 -gemma-3-1b-it,INT4-MIXED,1024,217.3,18.8,53.19 -gemma-3-1b-it,INT4-MIXED,32,34.9,18.8,53.19 -gemma-3-1b-it,INT4-MIXED,32,35,19,52.63 -minicpm4-0.5b,FP16,32,23.5,19.4,51.55 -gemma-3-1b-it,INT4-MIXED,1024,220.5,19.7,50.76 -gemma-3-1b-it,INT4-MIXED,1024,253.8,19.9,50.25 -minicpm-1b-sft,INT4-MIXED,31,57.6,20,50.00 -glm-edge-1.5b-chat,INT4-MIXED,32,48.5,20.3,49.26 -minicpm4-0.5b,FP16,1024,143.7,20.4,49.02 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,41.3,20.7,48.31 -minicpm-1b-sft,INT4-MIXED,31,45.4,21,47.62 -minicpm-1b-sft,INT4-MIXED,31,47.7,21.4,46.73 -tiny-llama-1.1b-chat,INT8-CW,32,39.9,21.4,46.73 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,45.5,21.7,46.08 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,45.3,21.8,45.87 -qwen2.5-1.5b-instruct,INT4-MIXED,32,44.7,21.8,45.87 -nanollava,FP16,760,285.1,21.9,45.66 -qwen2.5-coder-0.5b-instruct,FP16,32,27.5,22,45.45 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,45.4,22,45.45 -glm-edge-1.5b-chat,INT4-MIXED,1024,559.9,22.1,45.25 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,325.4,22.2,45.05 -qwen2.5-coder-0.5b-instruct,FP16,1024,156.5,22.5,44.44 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,357.9,23.2,43.10 -minicpm-1b-sft,INT4-MIXED,1014,358.6,23.2,43.10 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,359.1,23.2,43.10 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,360,23.2,43.10 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,377.7,23.4,42.74 -tiny-llama-1.1b-chat,INT8-CW,1024,272.3,23.4,42.74 -qwen2.5-1.5b-instruct,INT4-MIXED,32,47.9,23.5,42.55 -gemma-3-1b-it,INT8-CW,32,44.5,23.9,41.84 -nanollava,FP16,1752,512.2,23.9,41.84 -minicpm-1b-sft,INT4-MIXED,1014,369.9,24,41.67 -minicpm-1b-sft,INT4-MIXED,1014,420.4,24.7,40.49 -gemma-3-1b-it,INT8-CW,1024,258.7,24.8,40.32 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,381.7,24.8,40.32 -llama-3.2-1b-instruct,INT8-CW,32,44.2,25.2,39.68 -llama-3.2-1b-instruct,INT8-CW,1024,260.8,26.7,37.45 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,62.6,30.8,32.47 -qwen3.5-0.8b,FP16,85,100.3,31.3,31.95 -glm-edge-1.5b-chat,INT8-CW,32,62.8,31.7,31.55 -minicpm-1b-sft,INT8-CW,31,59,31.8,31.45 -qwen3.5-0.8b,FP16,1024,421.9,31.8,31.45 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,369.8,32.2,31.06 -deepseek-r1-distill-qwen-1.5b,INT8-CW,32,62.2,33.1,30.21 -qwen2.5-1.5b-instruct,INT8-CW,32,61.6,33.1,30.21 -qwen2.5-coder-1.5b-instruct,INT8-CW,32,62.6,33.2,30.12 -glm-edge-1.5b-chat,INT8-CW,1024,560.5,33.5,29.85 -gemma-2b-it,INT4-MIXED,32,54.6,33.7,29.67 -phi-2,INT4-MIXED,32,65,33.7,29.67 -stable-zephyr-3b-dpo,INT4-MIXED,32,63.6,34,29.41 -stablelm-3b-4e1t,INT4-MIXED,32,64.4,34,29.41 -deepseek-r1-distill-qwen-1.5b,INT8-CW,1024,359.4,34.5,28.99 -qwen2.5-1.5b-instruct,INT8-CW,1024,361.9,34.5,28.99 -qwen2.5-coder-1.5b-instruct,INT8-CW,1024,360.5,34.5,28.99 -minicpm-1b-sft,INT8-CW,1014,410.9,34.9,28.65 -gemma-2b-it,INT4-MIXED,1024,400.7,35.3,28.33 -gemma-2b-it,INT4-MIXED,32,66.3,35.4,28.25 -gemma-2-2b,INT4-MIXED,33,57.3,37,27.03 -gemma-2b-it,INT4-MIXED,1024,493.6,37,27.03 -whisper-large-v3,INT4-MIXED,prompt0,579.9,37.4,26.74 -stable-zephyr-3b-dpo,INT4-MIXED,32,78.8,38,26.32 -whisper-large-v3,INT4-MIXED,prompt1,642.5,38.2,26.18 -phi-2,INT4-MIXED,1024,770,38.4,26.04 -stable-zephyr-3b-dpo,INT4-MIXED,1024,835.6,38.7,25.84 -stablelm-3b-4e1t,INT4-MIXED,1024,837.3,38.8,25.77 -gemma-2-2b,INT4-MIXED,33,58.4,39.3,25.45 -llama-3.2-3b-instruct,INT4-MIXED,32,69.1,39.8,25.13 -phi-2,INT4-MIXED,32,84.2,40,25.00 -gemma-2-2b,INT4-MIXED,1025,539.7,40.1,24.94 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,71.5,40.2,24.88 -tiny-llama-1.1b-chat,FP16,32,49.9,40.7,24.57 -llama-3.2-3b-instruct,INT4-MIXED,32,82,40.9,24.45 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,81.6,40.9,24.45 -llama-3.2-3b-instruct,INT4-MIXED,32,81.9,41.5,24.10 -tiny-llama-1.1b-chat,FP16,1024,336.1,42.2,23.70 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,536.7,42.3,23.64 -gemma-2-2b,INT4-MIXED,1025,616.3,42.4,23.58 -stable-zephyr-3b-dpo,INT4-MIXED,1024,940.7,42.7,23.42 -llama-3.2-3b-instruct,INT4-MIXED,1024,589.4,42.8,23.36 -stablelm-3b-4e1t,INT4-MIXED,32,88.9,42.9,23.31 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,556,43.2,23.15 -gemma-3-1b-it,FP16,32,52.9,43.3,23.09 -whisper-large-v3,INT8-CW,prompt0,551.3,43.3,23.09 -whisper-large-v3,INT8-CW,prompt1,608.2,43.8,22.83 -gemma-3-1b-it,FP16,1024,307.9,43.9,22.78 -llama-3.2-3b-instruct,INT4-MIXED,1024,622.1,43.9,22.78 -phi-3-mini-4k-instruct,INT4-MIXED,32,80.2,44.2,22.62 -phi-3-mini-128k-instruct,INT4-MIXED,32,81.5,44.3,22.57 -phi-3.5-mini-instruct,INT4-MIXED,32,79.1,44.3,22.57 -llama-3.2-3b-instruct,INT4-MIXED,1024,706.3,44.4,22.52 -phi-2,INT4-MIXED,1024,1016.2,44.7,22.37 -phi-3-mini-4k-instruct,INT4-MIXED,32,96.5,45.6,21.93 -phi-3.5-mini-instruct,INT4-MIXED,32,96.8,45.7,21.88 -phi-3-mini-128k-instruct,INT4-MIXED,32,101.7,46.3,21.60 -gemma-4-e2b-it,INT4-MIXED,274,816.8,47,21.28 -stablelm-3b-4e1t,INT4-MIXED,1024,952.3,47.6,21.01 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,95.5,48,20.83 -llama-3.2-1b-instruct,FP16,32,57.7,48.4,20.66 -internvl2-4b,INT4-MIXED,297,496.7,48.8,20.49 -gemma-4-e2b-it,INT4-MIXED,274,870.6,48.9,20.45 -internvl2-4b,INT4-MIXED,297,535.7,49.3,20.28 -phi-4-mini-instruct,INT4-MIXED,32,85.9,49.6,20.16 -phi-4-mini-reasoning,INT4-MIXED,32,88.8,49.7,20.12 -llama-3.2-1b-instruct,FP16,1024,331.3,49.9,20.04 -qwen3-4b,INT4-MIXED,32,88.3,50.1,19.96 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,1029.7,50.3,19.88 -gemma-2b-it,INT8-CW,32,88.5,50.5,19.80 -phi-3-mini-4k-instruct,INT4-MIXED,1024,910.5,50.6,19.76 -phi-3.5-mini-instruct,INT4-MIXED,1024,911.3,50.6,19.76 -phi-3-mini-128k-instruct,INT4-MIXED,1024,911.1,50.7,19.72 -phi-4-mini-instruct,INT4-MIXED,32,102,50.8,19.69 -phi-4-mini-reasoning,INT4-MIXED,32,100.3,51,19.61 -phi-4-mini-reasoning,INT4-MIXED,32,101.8,51.3,19.49 -phi-3-mini-4k-instruct,INT4-MIXED,1024,948,51.9,19.27 -qwen3-4b,INT4-MIXED,32,104.1,51.9,19.27 -phi-3.5-mini-instruct,INT4-MIXED,1024,951.2,52.1,19.19 -gemma-2b-it,INT8-CW,1024,483.4,52.2,19.16 -gemma-3-4b-it,INT4-MIXED,32,96.1,52.3,19.12 -glm-edge-4b-chat,INT4-MIXED,32,119.6,52.6,19.01 -phi-3-mini-128k-instruct,INT4-MIXED,1024,1162.5,52.7,18.98 -phi-4-mini-instruct,INT4-MIXED,1024,905.2,52.9,18.90 -phi-3.5-vision-instruct,INT4-MIXED,802,1189,53,18.87 -phi-4-mini-reasoning,INT4-MIXED,1024,905.2,53,18.87 -gemma-3-4b-it,INT4-MIXED,32,105.3,53.8,18.59 -phi-3-mini-4k-instruct,INT4-MIXED,32,115.2,53.9,18.55 -afm-4.5b,INT4-MIXED,32,104.8,54.1,18.48 -phi-3.5-mini-instruct,INT4-MIXED,32,117.6,54.1,18.48 -gemma-3-4b-it,INT4-MIXED,32,106.2,54.2,18.45 -phi-4-mini-instruct,INT4-MIXED,1024,938.4,54.2,18.45 -phi-4-mini-reasoning,INT4-MIXED,1024,943.8,54.3,18.42 -internvl2-4b,INT4-MIXED,1027,1355.1,54.6,18.32 -phi-4-mini-reasoning,INT4-MIXED,1024,1019.6,54.7,18.28 -internvl2-4b,INT4-MIXED,1027,1381.9,54.9,18.21 -phi-3.5-vision-instruct,INT4-MIXED,1032,1423.9,54.9,18.21 -qwen3-4b,INT4-MIXED,1024,822.2,55,18.18 -phi-2,INT8-CW,32,105.5,55.7,17.95 -gemma-3-4b-it,INT4-MIXED,1024,869.7,55.9,17.89 -gemma-2-2b,INT8-CW,33,106,56,17.86 -stable-zephyr-3b-dpo,INT8-CW,32,106,56.3,17.76 -stablelm-3b-4e1t,INT8-CW,32,106.1,56.3,17.76 -qwen3-4b,INT4-MIXED,1024,860.8,56.5,17.70 -glm-edge-4b-chat,INT4-MIXED,1024,1419.2,56.7,17.64 -minicpm-1b-sft,FP16,31,68,56.8,17.61 -afm-4.5b,INT4-MIXED,1024,923.1,57.4,17.42 -gemma-3-4b-it,INT4-MIXED,1024,892.1,57.5,17.39 -gemma-3-4b-it,INT4-MIXED,1024,957.5,57.9,17.27 -glm-edge-1.5b-chat,FP16,32,75.7,58.5,17.09 -gemma-2-2b,INT8-CW,1025,680.4,59,16.95 -phi-3-mini-4k-instruct,INT4-MIXED,1024,1284.3,60.2,16.61 -minicpm-1b-sft,FP16,1014,494.4,60.3,16.58 -minicpm3-4b,INT4-MIXED,32,121.2,60.3,16.58 -phi-2,INT8-CW,1024,889,60.4,16.56 -phi-3.5-mini-instruct,INT4-MIXED,1024,1397.6,60.4,16.56 -stablelm-3b-4e1t,INT8-CW,1024,949.6,60.9,16.42 -stable-zephyr-3b-dpo,INT8-CW,1024,961.6,61,16.39 -glm-edge-1.5b-chat,FP16,1024,606.8,61.1,16.37 -qwen2.5-coder-3b-instruct,INT8-CW,32,112,61.6,16.23 -deepseek-r1-distill-qwen-1.5b,FP16,32,74.6,61.9,16.16 -qwen2.5-1.5b-instruct,FP16,32,75.3,61.9,16.16 -qwen2.5-coder-1.5b-instruct,FP16,32,74.9,62,16.13 -minicpm3-4b,INT4-MIXED,32,131.1,62.4,16.03 -minicpm3-4b,INT4-MIXED,32,134.2,63,15.87 -qwen2.5-1.5b-instruct,FP16,1024,452.8,63.2,15.82 -qwen2.5-coder-1.5b-instruct,FP16,1024,455.1,63.2,15.82 -deepseek-r1-distill-qwen-1.5b,FP16,1024,455.1,63.3,15.80 -phi-4-multimodal-instruct,INT4-MIXED,578,1062.4,63.6,15.72 -qwen2.5-coder-3b-instruct,INT8-CW,1024,668.2,63.7,15.70 -whisper-large-v3,FP16,prompt0,588.1,64.1,15.60 -phi-4-multimodal-instruct,INT4-MIXED,786,1271.9,64.3,15.55 -whisper-large-v3,FP16,prompt1,657.1,64.4,15.53 -chatglm3-6b,INT4-MIXED,32,112.2,65.4,15.29 -llama-3.2-3b-instruct,INT8-CW,32,114.2,65.7,15.22 -phi-4-multimodal-instruct,INT4-MIXED,1362,2567.3,66.4,15.06 -gemma-4-e2b-it,INT4-MIXED,1024,1346.4,66.5,15.04 -phi-4-multimodal-instruct,INT4-MIXED,1570,2829.5,67,14.93 -chatglm3-6b,INT4-MIXED,32,141.6,68.4,14.62 -gemma-4-e2b-it,INT4-MIXED,1024,1410,68.6,14.58 -gemma-4-e2b-it,INT8-CW,274,936.3,68.6,14.58 -llama-3.2-3b-instruct,INT8-CW,1024,740.9,68.6,14.58 -chatglm3-6b,INT4-MIXED,1024,1166.7,68.8,14.53 -chatglm3-6b,INT4-MIXED,1024,1265.5,71.3,14.03 -minicpm3-4b,INT4-MIXED,1024,2123,72.8,13.74 -llama-2-7b-chat-hf,INT4-MIXED,32,128.7,72.9,13.72 -codellama-7b,INT4-MIXED,32,124.7,73.2,13.66 -minicpm3-4b,INT4-MIXED,1024,2173.9,74.9,13.35 -qwen3-vl-4b-instruct,INT4-MIXED,4907,14047.4,75.4,13.26 -qwen3-vl-4b-instruct,INT4-MIXED,4937,14752.4,75.4,13.26 -biomistral-7b-slerp,INT4-MIXED,7,84.6,75.5,13.25 -llama-2-7b-chat-hf,INT4-MIXED,32,157.6,75.5,13.25 -minicpm3-4b,INT4-MIXED,1024,2302.7,75.5,13.25 -qwen3_8b_eagle3,INT4-MIXED,32,192.1,75.9,13.18 -falcon-7b-instruct,INT4-MIXED,32,134.9,76.1,13.14 -phi-4-multimodal-instruct,INT4-MIXED,578,1310.8,76.1,13.14 -stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,81.5,76.1,13.14 -mistral-7b-instruct-v0.2,INT4-MIXED,32,135.2,76.2,13.12 -neural-chat-7b-v3-3,INT4-MIXED,32,135.7,76.2,13.12 -zephyr-7b-beta,INT4-MIXED,32,134,76.3,13.11 -mistral-7b-instruct-v0.3,INT4-MIXED,32,137.2,76.4,13.09 -codellama-7b,INT4-MIXED,32,158.8,76.5,13.07 -phi-4-multimodal-instruct,INT4-MIXED,786,1558.9,76.8,13.02 -phi-3-mini-128k-instruct,INT8-CW,32,142.2,77,12.99 -qwen3-vl-4b-instruct,INT4-MIXED,4907,14219.3,77,12.99 -phi-3-mini-4k-instruct,INT8-CW,32,144.4,77.1,12.97 -phi-3.5-mini-instruct,INT8-CW,32,144.1,77.1,12.97 -qwen3-vl-4b-instruct,INT4-MIXED,4937,14906.6,77.1,12.97 -qwen3-vl-4b-instruct,INT4-MIXED,4937,15704.7,78,12.82 -qwen3-vl-4b-instruct,INT4-MIXED,4907,14768.1,78.1,12.80 -mistral-7b-instruct-v0.3,INT4-MIXED,32,169,78.7,12.71 -phi-4-mini-instruct,INT8-CW,32,138.5,78.7,12.71 -mistral-7b-instruct-v0.2,INT4-MIXED,32,168.9,79,12.66 -phi-4-mini-reasoning,INT8-CW,32,139.1,79,12.66 -neural-chat-7b-v3-3,INT4-MIXED,32,171.3,79.5,12.58 -mistral-7b-instruct-v0.2,INT4-MIXED,32,173.6,79.6,12.56 -mistral-7b-instruct-v0.3,INT4-MIXED,32,172.5,79.6,12.56 -mistral-7b-instruct-v0.1,INT4-MIXED,32,173.5,79.7,12.55 -llama-2-7b-chat-hf,INT4-MIXED,1024,1176.4,79.8,12.53 -phi-4-multimodal-instruct,INT4-MIXED,1362,3183,79.9,12.52 -codellama-7b,INT4-MIXED,1024,1180.7,80,12.50 -falcon-7b-instruct,INT4-MIXED,1024,1270.5,80,12.50 -gemma-3-4b-it,INT8-CW,32,152.9,80,12.50 -qwen2.5-7b-instruct,INT4-MIXED,32,151.9,80.1,12.48 -qwen2.5-7b-instruct-1m,INT4-MIXED,32,146.9,80.1,12.48 -phi-4-multimodal-instruct,INT4-MIXED,1570,3482.3,80.2,12.47 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,1270.3,80.4,12.44 -zephyr-7b-beta,INT4-MIXED,1024,1281.7,80.4,12.44 -stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,83.7,80.5,12.42 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,1261.5,80.7,12.39 -neural-chat-7b-v3-3,INT4-MIXED,1024,1267,80.7,12.39 -qwen3-4b,INT8-CW,32,147.6,81.4,12.29 -biomistral-7b-slerp,INT4-MIXED,7,91.1,81.7,12.24 -minicpm-v-2_6,INT4-MIXED,228,1203.8,81.7,12.24 -minicpm-o-2_6,INT4-MIXED,238,1197.9,81.9,12.21 -internvl2-4b,INT8-CW,297,624,82,12.20 -phi-4-mini-instruct,INT8-CW,1024,1055.2,82,12.20 -llama-2-7b-chat-hf,INT4-MIXED,1024,1282.3,82.3,12.15 -phi-4-mini-reasoning,INT8-CW,1024,1077,82.4,12.14 -qwen2-7b-instruct,INT4-MIXED,32,170.7,82.6,12.11 -qwen2.5-7b-instruct,INT4-MIXED,32,173.4,82.7,12.09 -deepseek-r1-distill-qwen-7b,INT4-MIXED,32,172.1,82.8,12.08 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,1360.8,82.9,12.06 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,1358.3,83.1,12.03 -qwen2.5-7b-instruct,INT4-MIXED,1024,1228.6,83.1,12.03 -phi-3-mini-128k-instruct,INT8-CW,1024,1227.5,83.2,12.02 -qwen2.5-7b-instruct-1m,INT4-MIXED,1024,1225.6,83.2,12.02 -llama-3-8b-instruct,INT4-MIXED,32,143.7,83.3,12.00 -llama-3.1-8b-instruct,INT4-MIXED,32,147.7,83.3,12.00 -codellama-7b,INT4-MIXED,1024,1412.5,83.4,11.99 -deepseek-r1-distill-qwen-7b,INT4-MIXED,32,174.2,83.4,11.99 -phi-3-mini-4k-instruct,INT8-CW,1024,1227.1,83.4,11.99 -phi-3.5-mini-instruct,INT8-CW,1024,1214.6,83.4,11.99 -qwen2.5-7b-instruct-1m,INT4-MIXED,32,176.1,83.4,11.99 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,1546.4,83.6,11.96 -phi-3.5-vision-instruct,INT8-CW,802,1269.2,83.7,11.95 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,143.7,83.8,11.93 -gemma-3-4b-it,INT8-CW,1024,1020,83.8,11.93 -qwen2-7b-instruct,INT4-MIXED,32,176,83.8,11.93 -qwen2.5-7b-instruct,INT4-MIXED,32,176.4,83.8,11.93 -neural-chat-7b-v3-3,INT4-MIXED,1024,1544.8,83.9,11.92 -mistral-7b-instruct-v0.1,INT4-MIXED,1025,1667.5,84,11.90 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,1535.1,84,11.90 -minicpm-v-2_6,INT4-MIXED,228,1231.1,84.3,11.86 -qwen3_8b_eagle3,INT4-MIXED,1024,1632.2,84.7,11.81 -phi-3.5-vision-instruct,INT8-CW,1032,1567.2,85.3,11.72 -qwen2-7b-instruct,INT4-MIXED,1024,1303.6,85.5,11.70 -deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,1302.8,85.6,11.68 -minicpm-v-2_6,INT4-MIXED,228,1232.2,85.6,11.68 -minicpm-o-2_6,INT4-MIXED,238,1227.5,85.7,11.67 -qwen2.5-7b-instruct,INT4-MIXED,1024,1294.4,85.7,11.67 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,175.4,85.9,11.64 -llama-3-8b-instruct,INT4-MIXED,32,178.4,86,11.63 -qwen3-8b,INT4-MIXED,32,146.8,86,11.63 -llama-3-8b-instruct,INT4-MIXED,32,175.9,86.1,11.61 -llama-3.1-8b-instruct,INT4-MIXED,32,174.7,86.1,11.61 -qwen3-4b,INT8-CW,1024,997.8,86.1,11.61 -minicpm4-8b,INT4-MIXED,32,157.6,86.3,11.59 -deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,1480.4,86.4,11.57 -qwen2.5-7b-instruct-1m,INT4-MIXED,1024,1475.2,86.4,11.57 -qwen2.5-vl-7b-instruct,INT4-MIXED,32,239.3,86.4,11.57 -qwen2.5-7b-instruct,INT4-MIXED,1024,1482.6,86.5,11.56 -qwen2-7b-instruct,INT4-MIXED,1024,1486.3,86.9,11.51 -glm-edge-4b-chat,INT8-CW,32,166.8,87,11.49 -llama-3-8b-instruct,INT4-MIXED,32,179,87,11.49 -falcon-7b-instruct,INT4-MIXED,32,186.2,87.6,11.42 -minicpm-v-4_5,INT4-MIXED,217,1233.2,87.6,11.42 -llama-3.1-8b-instruct,INT4-MIXED,1024,1274.9,87.7,11.40 -llama-3-8b-instruct,INT4-MIXED,1024,1279.6,87.8,11.39 -bloomz-7b1,INT4-MIXED,32,140.6,87.9,11.38 -deepseek-r1-distill-llama-8b,INT4-MIXED,1024,1305.5,87.9,11.38 -qwen3-8b,INT4-MIXED,32,184.7,88.8,11.26 -gemma-4-e2b-it,INT8-CW,1024,1452,89,11.24 -minicpm4-8b,INT4-MIXED,32,190.2,89.1,11.22 -qwen2.5-vl-7b-instruct,INT4-MIXED,1024,1400.2,89.3,11.20 -afm-4.5b,INT8-CW,32,168.6,89.4,11.19 -internvl2-4b,INT8-CW,1027,1554.7,89.4,11.19 -qwen2.5-vl-7b-instruct,INT4-MIXED,32,257.1,89.4,11.19 -fara-7b,INT4-MIXED,32,271.7,89.8,11.14 -qwen2.5-vl-7b-instruct,INT4-MIXED,32,258.9,90,11.11 -minicpm4-8b,INT4-MIXED,32,190,90.1,11.10 -deepseek-r1-distill-llama-8b,INT4-MIXED,1024,1377,90.3,11.07 -llama-3-8b-instruct,INT4-MIXED,1024,1370.4,90.3,11.07 -llama-3.1-8b-instruct,INT4-MIXED,1024,1369.7,90.3,11.07 -bloomz-7b1,INT4-MIXED,32,170.8,90.4,11.06 -llama-3-8b-instruct,INT4-MIXED,1024,1481.2,90.4,11.06 -minicpm4-8b,INT4-MIXED,1024,1345.2,90.4,11.06 -zephyr-7b-beta,INT4-MIXED,32,181.7,90.6,11.04 -glm-edge-4b-chat,INT8-CW,1024,1488.2,91.1,10.98 -llama-3-8b-instruct,INT4-MIXED,1024,1562.5,91.3,10.95 -minicpm-v-4_5,INT4-MIXED,217,1550.6,91.4,10.94 -qwen3-8b,INT4-MIXED,1024,1310.6,91.4,10.94 -falcon-7b-instruct,INT4-MIXED,1024,1604.6,91.6,10.92 -qwen3-8b,INT4-MIXED,32,189.4,91.9,10.88 -qwen2.5-vl-7b-instruct,INT4-MIXED,1024,1465.9,92.2,10.85 -afm-4.5b,INT8-CW,1024,986.9,92.8,10.78 -minicpm4-8b,INT4-MIXED,1024,1465.9,93,10.75 -qwen2.5-vl-7b-instruct,INT4-MIXED,1024,1640,93.1,10.74 -fara-7b,INT4-MIXED,1024,1974.8,93.4,10.71 -qwen3-8b,INT4-MIXED,1024,1395.6,94,10.64 -bloomz-7b1,INT4-MIXED,1024,1606.9,94.2,10.62 -minicpm4-8b,INT4-MIXED,1024,1677.2,94.3,10.60 -minicpm3-4b,INT8-CW,32,181,94.4,10.59 -qwen3.5-9b,INT4-MIXED,83,334,94.8,10.55 -zephyr-7b-beta,INT4-MIXED,1024,1418.2,94.9,10.54 -phi-4-multimodal-instruct,INT8-CW,578,1249.2,96.4,10.37 -qwen3.5-9b,INT4-MIXED,1024,2018,96.4,10.37 -phi-4-multimodal-instruct,INT8-CW,786,1510.7,96.7,10.34 -qwen3-8b,INT4-MIXED,1024,1622.1,96.8,10.33 -bloomz-7b1,INT4-MIXED,1024,1927.3,96.9,10.32 -glm-4-9b-chat-hf,INT4-MIXED,32,168.3,98.1,10.19 -gemma-2b-it,FP16,32,115.3,98.9,10.11 -baichuan2-7b-chat,INT4-MIXED,32,192.8,99.3,10.07 -gemma-2b-it,FP16,1024,654.6,99.7,10.03 -phi-4-multimodal-instruct,INT8-CW,1362,2977,100,10.00 \ No newline at end of file +minicpm4-0.5b,INT4-MIXED,32,22,7.3,136.9863014 +minicpm4-0.5b,INT4-MIXED,32,21.1,7.5,133.3333333 +minicpm4-0.5b,INT4-MIXED,32,23,7.7,129.8701299 +gemma-3-270m,INT4-MIXED,1024,67.1,7.9,126.5822785 +gemma-3-270m,INT4-MIXED,32,23.2,8,125 +lfm2.5-350m,INT4-MIXED,32,24.7,8.2,121.9512195 +gemma-3-270m,INT8-CW,32,22.9,8.3,120.4819277 +lfm2.5-350m,INT4-MIXED,1024,155,8.4,119.047619 +minicpm4-0.5b,INT4-MIXED,1024,100.2,8.4,119.047619 +gemma-3-270m,INT8-CW,1024,63.2,8.7,114.9425287 +minicpm4-0.5b,INT4-MIXED,1024,131.5,8.7,114.9425287 +minicpm4-0.5b,INT4-MIXED,1024,103.7,8.7,114.9425287 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,22.4,8.9,112.3595506 +nuextract-tiny,INT4-MIXED,32,23.7,9,111.1111111 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,22.5,9.1,109.8901099 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,23.2,9.2,108.6956522 +lfm2.5-350m,INT8-CW,32,26.2,9.7,103.0927835 +nuextract-tiny,INT4-MIXED,1024,125.7,9.8,102.0408163 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,105,9.8,102.0408163 +lfm2.5-350m,INT8-CW,1024,145.1,10,100 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,108.5,10,100 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,134.9,10.1,99.00990099 +minicpm4-0.5b,INT8-CW,32,23,10.4,96.15384615 +whisper-small,INT4-MIXED,32,139.8,10.6,94.33962264 +whisper-small,INT4-MIXED,1024,208.6,11,90.90909091 +nuextract-tiny,INT8-CW,32,24.9,11.1,90.09009009 +whisper-small,INT4-MIXED,32,141.5,11.2,89.28571429 +whisper-small,INT4-MIXED,1024,200.3,11.2,89.28571429 +minicpm4-0.5b,INT8-CW,1024,116.1,11.5,86.95652174 +distil-large-v2,INT4-MIXED,32,441.9,11.6,86.20689655 +distil-large-v2,INT8-CW,32,411.3,11.7,85.47008547 +distil-large-v2,INT8-CW,1024,467.8,11.7,85.47008547 +whisper-small,INT4-MIXED,32,154.8,11.7,85.47008547 +whisper-small,INT4-MIXED,1024,225.2,11.7,85.47008547 +qwen2.5-coder-0.5b-instruct,INT8-CW,32,25.4,11.8,84.74576271 +distil-large-v2,INT4-MIXED,1024,496,11.9,84.03361345 +whisper-small,INT8-CW,32,150.7,11.9,84.03361345 +whisper-small,INT8-CW,1024,226.4,11.9,84.03361345 +gemma-3-270m,FP16,32,22.5,12.3,81.30081301 +whisper-large-v3-turbo,INT8-CW,1024,481.2,12.4,80.64516129 +nanollava,INT4-MIXED,760,249.8,12.5,80 +nuextract-tiny,INT8-CW,1024,110.1,12.6,79.36507937 +qwen2.5-coder-0.5b-instruct,INT8-CW,1024,124.3,12.6,79.36507937 +whisper-large-v3-turbo,INT4-MIXED,1024,527.3,12.7,78.74015748 +whisper-large-v3-turbo,INT8-CW,32,423.8,12.7,78.74015748 +gemma-3-270m,FP16,1024,72.2,12.8,78.125 +whisper-large-v3-turbo,INT4-MIXED,32,450.8,12.8,78.125 +tiny-llama-1.1b-chat,INT4-MIXED,32,28.7,13.1,76.33587786 +nanollava,INT8-CW,760,239.6,13.7,72.99270073 +minicpm5-1b,INT4-MIXED,32,28.2,13.9,71.94244604 +qwen3.5-0.8b,INT4-MIXED,85,61.5,14.1,70.92198582 +nanollava,INT4-MIXED,1752,481.1,14.3,69.93006993 +qwen3.5-0.8b,INT4-MIXED,85,65.1,14.3,69.93006993 +qwen3.5-0.8b,INT4-MIXED,1024,347,14.5,68.96551724 +whisper-small,FP16,32,139.7,14.7,68.02721088 +whisper-small,FP16,1024,212.9,14.7,68.02721088 +qwen3.5-0.8b,INT4-MIXED,85,70.4,14.8,67.56756757 +qwen3.5-0.8b,INT4-MIXED,1024,349.5,14.9,67.11409396 +tiny-llama-1.1b-chat,INT4-MIXED,1024,219.5,14.9,67.11409396 +qwen3.5-0.8b,INT4-MIXED,1024,404.5,15.2,65.78947368 +lfm2.5-1.2b-instruct,INT4-MIXED,32,34,15.4,64.93506494 +lfm2-1.2b,INT4-MIXED,32,33.7,15.5,64.51612903 +nanollava,INT8-CW,1752,458,15.7,63.69426752 +distil-large-v2,FP16,32,437.5,15.8,63.29113924 +distil-large-v2,FP16,1024,494.2,15.8,63.29113924 +tiny-llama-1.1b-chat,INT4-MIXED,32,34,15.8,63.29113924 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,333.7,15.9,62.89308176 +lfm2.5-1.2b-instruct,INT4-MIXED,32,40.4,15.9,62.89308176 +lfm2-1.2b,INT4-MIXED,1024,316.1,16,62.5 +lfm2-1.2b,INT4-MIXED,32,39.8,16,62.5 +minicpm5-1b,INT4-MIXED,1024,224.3,16,62.5 +lfm2-1.2b,INT4-MIXED,32,39.8,16.1,62.11180124 +lfm2.5-1.2b-instruct,INT4-MIXED,32,41.3,16.1,62.11180124 +llama-3.2-1b-instruct,INT4-MIXED,32,31.2,16.1,62.11180124 +lfm2.5-350m,FP16,32,28,16.2,61.72839506 +lfm2-1.2b,INT4-MIXED,1024,361.6,16.3,61.34969325 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,376.8,16.3,61.34969325 +llama-3.2-1b-instruct,INT4-MIXED,32,32.7,16.3,61.34969325 +lfm2.5-350m,FP16,1024,159.1,16.4,60.97560976 +whisper-large-v3-turbo,FP16,32,457.2,16.4,60.97560976 +whisper-large-v3-turbo,FP16,1024,515.6,16.4,60.97560976 +lfm2-1.2b,INT4-MIXED,1024,410.4,16.5,60.60606061 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,422.2,16.7,59.88023952 +gemma-3-1b-it,INT4-MIXED,32,33.1,17.1,58.47953216 +llama-3.2-1b-instruct,INT4-MIXED,1024,210.5,17.4,57.47126437 +exaone-4.0-1.2b,INT4-MIXED,32,33.5,17.5,57.14285714 +llama-3.2-1b-instruct,INT4-MIXED,1024,257.9,17.6,56.81818182 +tiny-llama-1.1b-chat,INT4-MIXED,1024,299.1,17.6,56.81818182 +qwen3.5-0.8b,INT8-CW,85,71.7,17.8,56.17977528 +gemma-3-1b-it,INT4-MIXED,32,34.5,17.9,55.86592179 +exaone-4.0-1.2b,INT4-MIXED,32,37.8,18,55.55555556 +gemma-3-1b-it,INT4-MIXED,1024,206.2,18.1,55.24861878 +exaone-4.0-1.2b,INT4-MIXED,32,38.1,18.2,54.94505495 +gemma-3-1b-it,INT4-MIXED,32,34.7,18.2,54.94505495 +qwen3.5-0.8b,INT8-CW,1024,363.7,18.6,53.76344086 +gemma-3-1b-it,INT4-MIXED,1024,209.8,19,52.63157895 +minicpm5-1b,INT8-CW,32,35.9,19.2,52.08333333 +gemma-3-1b-it,INT4-MIXED,1024,244.7,19.3,51.8134715 +minicpm4-0.5b,FP16,32,23.7,19.3,51.8134715 +nuextract-tiny,FP16,32,28.7,19.3,51.8134715 +minicpm-1b-sft,INT4-MIXED,31,55.6,19.4,51.54639175 +exaone-4.0-1.2b,INT4-MIXED,1024,336.1,19.9,50.25125628 +minicpm4-0.5b,FP16,1024,133.5,20.3,49.26108374 +exaone-4.0-1.2b,INT4-MIXED,1024,344.9,20.5,48.7804878 +exaone-4.0-1.2b,INT4-MIXED,1024,378.4,20.7,48.30917874 +minicpm5-1b,INT8-CW,1024,210.1,20.7,48.30917874 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,41.2,20.7,48.30917874 +minicpm-1b-sft,INT4-MIXED,31,45.2,20.8,48.07692308 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,43.2,21.3,46.94835681 +minicpm-1b-sft,INT4-MIXED,31,47.5,21.4,46.72897196 +tiny-llama-1.1b-chat,INT8-CW,32,38.2,21.4,46.72897196 +nuextract-tiny,FP16,1024,131.4,21.5,46.51162791 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,44.9,21.6,46.2962963 +qwen2.5-1.5b-instruct,INT4-MIXED,32,44.2,21.7,46.08294931 +nanollava,FP16,760,284.9,21.8,45.87155963 +qwen2.5-coder-0.5b-instruct,FP16,32,27.8,21.9,45.66210046 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,45.4,22.2,45.04504505 +qwen2.5-coder-0.5b-instruct,FP16,1024,143.1,22.4,44.64285714 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,292.2,22.4,44.64285714 +minicpm-1b-sft,INT4-MIXED,1014,323.1,22.6,44.24778761 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,344.9,23,43.47826087 +gemma-3-1b-it,INT8-CW,32,42.8,23.1,43.29004329 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,347.2,23.2,43.10344828 +tiny-llama-1.1b-chat,INT8-CW,1024,247.1,23.3,42.91845494 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,344.8,23.5,42.55319149 +qwen2.5-1.5b-instruct,INT4-MIXED,32,47.7,23.6,42.37288136 +nanollava,FP16,1752,508.9,23.9,41.84100418 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,359.9,23.9,41.84100418 +minicpm-1b-sft,INT4-MIXED,1014,335.7,24.2,41.32231405 +gemma-3-1b-it,INT8-CW,1024,245.3,24.3,41.15226337 +lfm2-8b-a1b,INT4-MIXED,32,381.5,24.5,40.81632653 +minicpm-1b-sft,INT4-MIXED,1014,389.3,24.6,40.6504065 +llama-3.2-1b-instruct,INT8-CW,32,41,24.7,40.48582996 +lfm2-1.2b,INT8-CW,32,50.9,25.2,39.68253968 +lfm2-8b-a1b,INT4-MIXED,1024,881.8,25.2,39.68253968 +lfm2.5-1.2b-instruct,INT8-CW,32,50.9,25.3,39.5256917 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,369.1,25.4,39.37007874 +lfm2-1.2b,INT8-CW,1024,410.1,25.9,38.61003861 +lfm2.5-1.2b-instruct,INT8-CW,1024,391.1,26,38.46153846 +llama-3.2-1b-instruct,INT8-CW,1024,243.1,26.2,38.16793893 +exaone-4.0-1.2b,INT8-CW,32,47.5,27.2,36.76470588 +exaone-4.0-1.2b,INT8-CW,1024,372.1,29.8,33.55704698 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,59.9,30,33.33333333 +minicpm-1b-sft,INT8-CW,31,56,31.1,32.15434084 +qwen3.5-0.8b,FP16,85,74.7,31.2,32.05128205 +qwen3.5-0.8b,FP16,1024,388.1,31.6,31.64556962 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,353.7,31.8,31.44654088 +deepseek-r1-distill-qwen-1.5b,INT8-CW,32,59.6,32.3,30.95975232 +qwen2.5-coder-1.5b-instruct,INT8-CW,32,62.4,32.9,30.39513678 +stable-zephyr-3b-dpo,INT4-MIXED,32,62.3,32.9,30.39513678 +qwen2.5-1.5b-instruct,INT8-CW,32,62.3,33,30.3030303 +phi-2,INT4-MIXED,32,64.7,33.4,29.94011976 +deepseek-r1-distill-qwen-1.5b,INT8-CW,1024,340.6,33.7,29.6735905 +minicpm-1b-sft,INT8-CW,1014,369.3,34.1,29.3255132 +qwen2.5-1.5b-instruct,INT8-CW,1024,350,34.3,29.15451895 +qwen2.5-coder-1.5b-instruct,INT8-CW,1024,350.6,34.3,29.15451895 +gemma-4-e2b-it,INT4-MIXED,274,720.1,34.7,28.8184438 +minicpm5-1b,FP16,32,44,36.3,27.54820937 +gemma-4-e2b-it,INT4-MIXED,274,758.7,36.4,27.47252747 +whisper-large-v3,INT4-MIXED,32,578.5,36.6,27.32240437 +stable-zephyr-3b-dpo,INT4-MIXED,32,77,37,27.02702703 +gemma-4-e2b-it,INT4-MIXED,1024,1171.1,37.4,26.73796791 +stable-zephyr-3b-dpo,INT4-MIXED,1024,787.5,37.5,26.66666667 +minicpm5-1b,FP16,1024,257.7,37.6,26.59574468 +phi-2,INT4-MIXED,1024,719.8,38,26.31578947 +whisper-large-v3,INT4-MIXED,1024,636.2,38.2,26.17801047 +llama-3.2-3b-instruct,INT4-MIXED,32,64.9,38.3,26.10966057 +gemma-4-e2b-it,INT4-MIXED,1024,1311,39,25.64102564 +llama-3.2-3b-instruct,INT4-MIXED,32,77.5,39.4,25.38071066 +smollm3-3b,INT4-MIXED,32,71.9,39.5,25.3164557 +phi-2,INT4-MIXED,32,84.8,39.7,25.18891688 +llama-3.2-3b-instruct,INT4-MIXED,32,78.4,40,25 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,71.6,40.4,24.75247525 +smollm3-3b,INT4-MIXED,32,82.3,40.7,24.57002457 +tiny-llama-1.1b-chat,FP16,32,50.2,40.7,24.57002457 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,82.5,41.1,24.33090024 +smollm3-3b,INT4-MIXED,32,83.5,41.1,24.33090024 +stable-zephyr-3b-dpo,INT4-MIXED,1024,886,41.6,24.03846154 +llama-3.2-3b-instruct,INT4-MIXED,1024,555.8,41.8,23.92344498 +tiny-llama-1.1b-chat,FP16,1024,315.5,42,23.80952381 +gemma-3-1b-it,FP16,32,52.5,42.3,23.64066194 +smollm3-3b,INT4-MIXED,1024,538.5,42.5,23.52941176 +llama-3.2-3b-instruct,INT4-MIXED,1024,595.4,43,23.25581395 +whisper-large-v3,INT8-CW,32,546.6,43.2,23.14814815 +gemma-3-1b-it,FP16,1024,293,43.4,23.04147465 +llama-3.2-3b-instruct,INT4-MIXED,1024,677.8,43.4,23.04147465 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,514.5,43.4,23.04147465 +smollm3-3b,INT4-MIXED,1024,576.5,43.7,22.88329519 +whisper-large-v3,INT8-CW,1024,609.2,43.7,22.88329519 +smollm3-3b,INT4-MIXED,1024,684.3,44.1,22.67573696 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,535.7,44.3,22.57336343 +phi-2,INT4-MIXED,1024,973.5,44.6,22.42152466 +lfm2.5-1.2b-instruct,FP16,32,63.6,47.6,21.00840336 +lfm2-1.2b,FP16,32,64.1,47.8,20.92050209 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,95.6,48,20.83333333 +lfm2.5-1.2b-instruct,FP16,1024,453.7,48.2,20.74688797 +lfm2-1.2b,FP16,1024,440.9,48.3,20.70393375 +llama-3.2-1b-instruct,FP16,32,58.4,48.4,20.66115702 +qwen3-4b,INT4-MIXED,32,88.8,49.2,20.32520325 +phi-4-mini-instruct,INT4-MIXED,32,86.6,49.8,20.08032129 +phi-4-mini-reasoning,INT4-MIXED,32,85.6,49.9,20.04008016 +llama-3.2-1b-instruct,FP16,1024,316.3,50,20 +gemma-3-4b-it,INT4-MIXED,32,83.9,50.2,19.92031873 +qwen3-4b,INT4-MIXED,32,103,50.8,19.68503937 +phi-4-mini-instruct,INT4-MIXED,32,99.6,51,19.60784314 +phi-4-mini-reasoning,INT4-MIXED,32,100.5,51.1,19.56947162 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,1003.6,51.1,19.56947162 +phi-4-mini-reasoning,INT4-MIXED,32,102.2,51.4,19.45525292 +afm-4.5b,INT4-MIXED,32,103.3,51.7,19.34235977 +gemma-3-4b-it,INT4-MIXED,32,98.3,51.8,19.30501931 +gemma-4-e2b-it,INT8-CW,274,838.5,52.1,19.19385797 +gemma-3-4b-it,INT4-MIXED,32,98.3,52.2,19.15708812 +phi-3.5-vision-instruct,INT4-MIXED,802,1187.9,53.1,18.83239171 +gemma-3-4b-it,INT4-MIXED,1024,848.8,53.8,18.58736059 +phi-4-mini-instruct,INT4-MIXED,1024,867.6,54,18.51851852 +phi-4-mini-reasoning,INT4-MIXED,1024,871,54,18.51851852 +phi-3.5-vision-instruct,INT4-MIXED,1032,1427.3,54.8,18.24817518 +exaone-4.0-1.2b,FP16,32,64.5,55,18.18181818 +phi-4-mini-instruct,INT4-MIXED,1024,910.6,55.2,18.11594203 +gemma-3-4b-it,INT4-MIXED,1024,877.6,55.3,18.08318264 +gemma-4-e2b-it,INT8-CW,1024,1354.4,55.3,18.08318264 +afm-4.5b,INT4-MIXED,1024,888,55.4,18.05054152 +qwen3-4b,INT4-MIXED,1024,775.1,55.4,18.05054152 +phi-4-mini-reasoning,INT4-MIXED,1024,913.9,55.5,18.01801802 +phi-2,INT8-CW,32,105.9,55.6,17.98561151 +phi-4-mini-reasoning,INT4-MIXED,1024,985.3,55.6,17.98561151 +gemma-3-4b-it,INT4-MIXED,1024,946.1,55.8,17.92114695 +minicpm-1b-sft,FP16,31,68.3,56.3,17.76198934 +qwen3-4b,INT4-MIXED,1024,814.6,56.9,17.57469244 +exaone-4.0-1.2b,FP16,1024,449,57.6,17.36111111 +phi-4-mini-instruct,INT4-MIXED,32,116.1,57.9,17.27115717 +minicpm3-4b,INT4-MIXED,32,125,58.8,17.00680272 +minicpm-1b-sft,FP16,1014,453.8,59.4,16.83501684 +phi-2,INT8-CW,1024,842.8,60.3,16.58374793 +minicpm3-4b,INT4-MIXED,32,132.2,60.7,16.47446458 +deepseek-r1-distill-qwen-1.5b,FP16,32,73.8,60.9,16.42036125 +minicpm3-4b,INT4-MIXED,32,136,61.4,16.28664495 +qwen2.5-coder-1.5b-instruct,FP16,32,74.6,61.6,16.23376623 +qwen2.5-1.5b-instruct,FP16,32,74.3,61.7,16.20745543 +smollm3-3b,INT8-CW,32,111.8,61.7,16.20745543 +phi-4-mini-instruct,INT4-MIXED,1024,1197,62,16.12903226 +qwen2.5-coder-3b-instruct,INT8-CW,32,111.3,62,16.12903226 +deepseek-r1-distill-qwen-1.5b,FP16,1024,435.6,62.3,16.05136437 +qwen2.5-coder-1.5b-instruct,FP16,1024,442.5,62.9,15.89825119 +qwen2.5-1.5b-instruct,FP16,1024,440,63,15.87301587 +phi-4-multimodal-instruct,INT4-MIXED,578,1039.3,63.4,15.77287066 +phi-4-multimodal-instruct,INT4-MIXED,786,1257.1,64.1,15.60062402 +smollm3-3b,INT8-CW,1024,671.4,64.1,15.60062402 +qwen2.5-coder-3b-instruct,INT8-CW,1024,652.2,64.2,15.57632399 +whisper-large-v3,FP16,32,583.8,64.2,15.57632399 +llama-3.2-3b-instruct,INT8-CW,32,112.1,64.4,15.52795031 +whisper-large-v3,FP16,1024,656,64.6,15.47987616 +phi-4-multimodal-instruct,INT4-MIXED,1362,2538.4,66.2,15.10574018 +phi-4-multimodal-instruct,INT4-MIXED,1570,2811.8,67,14.92537313 +llama-3.2-3b-instruct,INT8-CW,1024,714.5,67.3,14.85884101 +minicpm3-4b,INT4-MIXED,1024,2139.5,67.5,14.81481481 +codellama-7b,INT4-MIXED,32,122.8,68.9,14.5137881 +minicpm3-4b,INT4-MIXED,1024,2186.1,69.3,14.43001443 +minicpm3-4b,INT4-MIXED,1024,2315.3,70,14.28571429 +llama-2-7b-chat-hf,INT4-MIXED,32,121.9,70.1,14.26533524 +biomistral-7b-slerp,INT4-MIXED,7,81.8,72.3,13.83125864 +codellama-7b,INT4-MIXED,32,156.6,72.4,13.8121547 +llama-2-7b-chat-hf,INT4-MIXED,32,154.8,72.9,13.71742112 +neural-chat-7b-v3-3,INT4-MIXED,32,137.5,73.7,13.56852103 +zephyr-7b-beta,INT4-MIXED,32,133.2,74.1,13.49527665 +falcon-7b-instruct,INT4-MIXED,32,131.7,74.3,13.4589502 +codellama-7b,INT4-MIXED,1024,1134.9,74.4,13.44086022 +llama-2-7b-chat-hf,INT4-MIXED,1024,1132.6,75.8,13.19261214 +biomistral-7b-slerp,INT4-MIXED,7,85.8,75.9,13.17523057 +phi-4-multimodal-instruct,INT4-MIXED,578,1330.4,76,13.15789474 +qwen3-vl-4b-instruct,INT4-MIXED,4937,14679.5,76,13.15789474 +mistral-7b-instruct-v0.2,INT4-MIXED,32,134.5,76.3,13.1061599 +mistral-7b-instruct-v0.3,INT4-MIXED,32,137.9,76.3,13.1061599 +qwen3-vl-4b-instruct,INT4-MIXED,4907,13865.4,76.4,13.08900524 +phi-4-multimodal-instruct,INT4-MIXED,786,1585.9,76.5,13.07189542 +qwen3-vl-4b-instruct,INT4-MIXED,4907,14153.6,76.6,13.05483029 +qwen3-vl-4b-instruct,INT4-MIXED,4937,14954.7,76.7,13.03780965 +qwen3-vl-4b-instruct,INT4-MIXED,4937,15633.3,77.4,12.91989664 +stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,81.4,77.5,12.90322581 +qwen3-vl-4b-instruct,INT4-MIXED,4907,14616.5,77.6,12.88659794 +codellama-7b,INT4-MIXED,1024,1369.1,78,12.82051282 +llama-2-7b-chat-hf,INT4-MIXED,1024,1236.8,78.5,12.7388535 +phi-4-mini-instruct,INT8-CW,32,138.7,78.7,12.7064803 +phi-4-mini-reasoning,INT8-CW,32,140.4,78.8,12.69035533 +mistral-7b-instruct-v0.2,INT4-MIXED,32,171.1,78.9,12.67427123 +mistral-7b-instruct-v0.3,INT4-MIXED,32,168.6,79.1,12.64222503 +neural-chat-7b-v3-3,INT4-MIXED,1024,1232.9,79.1,12.64222503 +neural-chat-7b-v3-3,INT4-MIXED,32,166.8,79.4,12.59445844 +zephyr-7b-beta,INT4-MIXED,1024,1209.6,79.5,12.57861635 +gemma-3-4b-it,INT8-CW,32,143.8,79.6,12.56281407 +phi-4-multimodal-instruct,INT4-MIXED,1362,3223.1,79.7,12.54705144 +falcon-7b-instruct,INT4-MIXED,1024,1185.7,79.8,12.53132832 +mistral-7b-instruct-v0.1,INT4-MIXED,32,171,79.8,12.53132832 +mistral-7b-instruct-v0.2,INT4-MIXED,32,171.2,79.8,12.53132832 +mistral-7b-instruct-v0.3,INT4-MIXED,32,171.8,79.8,12.53132832 +phi-4-multimodal-instruct,INT4-MIXED,1570,3470.5,79.9,12.51564456 +deepseek-r1-distill-llama-8b,INT4-MIXED,32,142.8,80.4,12.43781095 +qwen2.5-7b-instruct-1m,INT4-MIXED,32,150.4,80.4,12.43781095 +deepseek-r1-distill-qwen-7b,INT4-MIXED,32,169.1,80.5,12.42236025 +qwen2.5-7b-instruct,INT4-MIXED,32,147.4,80.6,12.40694789 +minicpm-v-2_6,INT4-MIXED,228,1189.5,81.3,12.300123 +deepseek-r1-distill-qwen-7b,INT4-MIXED,32,174,81.4,12.28501229 +qwen3-4b,INT8-CW,32,145.4,81.5,12.26993865 +minicpm-o-2_6,INT4-MIXED,238,1190.3,81.6,12.25490196 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,1238.6,81.6,12.25490196 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,1221.5,81.7,12.23990208 +stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,83.4,81.7,12.23990208 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,178.9,81.8,12.22493888 +llama-3.1-8b-instruct,INT4-MIXED,32,142.3,81.9,12.21001221 +phi-4-mini-instruct,INT8-CW,1024,1030.3,82,12.19512195 +phi-4-mini-reasoning,INT8-CW,1024,1038.4,82.1,12.18026797 +deepseek-r1-distill-llama-8b,INT4-MIXED,32,173,83.1,12.03369434 +qwen2-7b-instruct,INT4-MIXED,32,172.2,83.2,12.01923077 +qwen2.5-7b-instruct,INT4-MIXED,32,172.2,83.2,12.01923077 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,205.5,83.2,12.01923077 +gemma-3-4b-it,INT8-CW,1024,1010.3,83.3,12.00480192 +llama-3-8b-instruct,INT4-MIXED,32,152.8,83.3,12.00480192 +bloomz-7b1,INT4-MIXED,32,137.9,83.6,11.96172249 +phi-3.5-vision-instruct,INT8-CW,802,1284.4,84,11.9047619 +qwen2.5-7b-instruct-1m,INT4-MIXED,32,177.5,84,11.9047619 +qwen2-7b-instruct,INT4-MIXED,32,175.3,84.1,11.89060642 +qwen2.5-7b-instruct,INT4-MIXED,32,176.3,84.1,11.89060642 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,1324.9,84.3,11.8623962 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,1316.3,84.4,11.84834123 +deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,1248.9,84.5,11.83431953 +llama-3-8b-instruct,INT4-MIXED,32,177.3,84.5,11.83431953 +minicpm-v-2_6,INT4-MIXED,228,1232.1,84.5,11.83431953 +llama-3-8b-instruct,INT4-MIXED,32,175,84.6,11.82033097 +llama-3.1-8b-instruct,INT4-MIXED,32,175,84.6,11.82033097 +neural-chat-7b-v3-3,INT4-MIXED,1024,1501.5,84.6,11.82033097 +qwen2.5-7b-instruct,INT4-MIXED,1024,1182,84.6,11.82033097 +qwen2.5-7b-instruct-1m,INT4-MIXED,1024,1183.9,84.6,11.82033097 +fara-7b,INT4-MIXED,32,218.4,84.9,11.77856302 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,1380.4,85,11.76470588 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,1505,85.1,11.75088132 +llama-3-8b-instruct,INT4-MIXED,32,178.6,85.3,11.72332943 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,1498.6,85.3,11.72332943 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,207,85.3,11.72332943 +qwen3-8b,INT4-MIXED,32,144.7,85.3,11.72332943 +deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,1430.7,85.5,11.69590643 +mistral-7b-instruct-v0.1,INT4-MIXED,1025,1610.8,85.5,11.69590643 +deepseek-r1-distill-llama-8b,INT4-MIXED,1024,1220.4,85.6,11.68224299 +minicpm-o-2_6,INT4-MIXED,238,1207.1,85.6,11.68224299 +minicpm-v-2_6,INT4-MIXED,228,1354.5,85.7,11.66861144 +phi-3.5-vision-instruct,INT8-CW,1032,1568.1,85.8,11.65501166 +qwen3-4b,INT8-CW,1024,954.6,86.3,11.58748552 +falcon-7b-instruct,INT4-MIXED,32,187.1,86.6,11.54734411 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,1394.6,86.8,11.52073733 +afm-4.5b,INT8-CW,32,156.7,87,11.49425287 +minicpm4-8b,INT4-MIXED,32,154.5,87,11.49425287 +qwen2-7b-instruct,INT4-MIXED,1024,1263.4,87.1,11.48105626 +llama-3.1-8b-instruct,INT4-MIXED,1024,1220.1,87.2,11.46788991 +qwen2.5-7b-instruct,INT4-MIXED,1024,1256,87.2,11.46788991 +minicpm-v-4_5,INT4-MIXED,217,1237.4,87.5,11.42857143 +bloomz-7b1,INT4-MIXED,32,162.6,87.7,11.40250855 +fara-7b,INT4-MIXED,1024,1898.9,87.8,11.38952164 +qwen2-7b-instruct,INT4-MIXED,1024,1438.9,88,11.36363636 +qwen2.5-7b-instruct-1m,INT4-MIXED,1024,1437.4,88,11.36363636 +qwen2.5-7b-instruct,INT4-MIXED,1024,1439.6,88.1,11.3507378 +qwen3-8b,INT4-MIXED,32,182.2,88.1,11.3507378 +deepseek-r1-distill-llama-8b,INT4-MIXED,1024,1324.6,88.4,11.31221719 +llama-3-8b-instruct,INT4-MIXED,1024,1252.8,88.5,11.29943503 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,1579.7,88.6,11.28668172 +bloomz-7b1,INT4-MIXED,1024,1602.7,89.1,11.22334456 +qwen3-8b,INT4-MIXED,32,186.5,89.7,11.14827202 +llama-3-8b-instruct,INT4-MIXED,1024,1314.9,89.9,11.12347052 +llama-3-8b-instruct,INT4-MIXED,1024,1353.3,89.9,11.12347052 +llama-3.1-8b-instruct,INT4-MIXED,1024,1323.5,90,11.11111111 +afm-4.5b,INT8-CW,1024,947.5,90.3,11.07419712 +zephyr-7b-beta,INT4-MIXED,32,181.7,90.5,11.04972376 +llama-3-8b-instruct,INT4-MIXED,1024,1497.3,90.7,11.02535832 +minicpm4-8b,INT4-MIXED,32,196,90.7,11.02535832 +minicpm-v-4_5,INT4-MIXED,217,1295.8,91.3,10.95290252 +qwen3-8b,INT4-MIXED,1024,1267,91.3,10.95290252 +falcon-7b-instruct,INT4-MIXED,1024,1538.5,92.1,10.8577633 +minicpm4-8b,INT4-MIXED,32,190,92.1,10.8577633 +minicpm4-8b,INT4-MIXED,1024,1285.7,92.3,10.83423619 +bloomz-7b1,INT4-MIXED,1024,1935.5,93.3,10.71811361 +qwen3-8b,INT4-MIXED,1024,1376.4,94.1,10.62699256 +minicpm3-4b,INT8-CW,32,180.1,94.3,10.60445387 +qwen3_8b_draft_eagle3,INT4-MIXED,32,192.3,94.4,10.59322034 +qwen3-8b,INT4-MIXED,1024,1579,95.6,10.46025105 +zephyr-7b-beta,INT4-MIXED,1024,1398,95.6,10.46025105 +qwen3.5-9b,INT4-MIXED,83,306.6,95.9,10.42752868 +minicpm4-8b,INT4-MIXED,1024,1622.4,96,10.41666667 +gemma-7b-it,INT4-MIXED,32,175,96.2,10.3950104 +phi-4-multimodal-instruct,INT8-CW,578,1263.3,96.4,10.37344398 +phi-4-multimodal-instruct,INT8-CW,786,1515,96.9,10.31991744 +minicpm4-8b,INT4-MIXED,1024,1424.4,97.5,10.25641026 +qwen3.5-9b,INT4-MIXED,1024,1904.6,97.5,10.25641026 +gemma-7b-it,INT4-MIXED,32,208.8,100,10 +phi-4-multimodal-instruct,INT8-CW,1362,3044.9,100,10 \ No newline at end of file diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-255H.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-255H.csv index b8f5ca4e367005..5b748aefd00700 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-255H.csv +++ b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-255H.csv @@ -1,386 +1,423 @@ Topology,Precision,Input Size,1st latency (ms),2nd latency (ms),2nd token per sec -distil-large-v2,INT4-MIXED,prompt0,186.5,6.4,156.25 -distil-large-v2,INT8-CW,prompt0,174.8,6.4,156.25 -distil-large-v2,INT8-CW,prompt1,218,6.4,156.25 -distil-large-v2,INT4-MIXED,prompt1,226.6,6.5,153.85 -whisper-large-v3-turbo,INT4-MIXED,prompt0,243.8,9.1,109.89 -whisper-large-v3-turbo,INT4-MIXED,prompt1,311.4,9.2,108.70 -distil-large-v2,FP16,prompt0,217.1,9.4,106.38 -whisper-large-v3-turbo,INT8-CW,prompt1,310.4,9.4,106.38 -distil-large-v2,FP16,prompt1,267.3,9.5,105.26 -whisper-large-v3-turbo,INT8-CW,prompt0,226.8,9.8,102.04 -minicpm4-0.5b,INT4-MIXED,32,36.1,10.5,95.24 -minicpm4-0.5b,INT4-MIXED,32,32.6,10.7,93.46 -minicpm4-0.5b,INT4-MIXED,1024,71.1,11.1,90.09 -minicpm4-0.5b,INT4-MIXED,1024,74.5,11.3,88.50 -minicpm4-0.5b,INT8-CW,32,33.2,11.5,86.96 -minicpm4-0.5b,INT4-MIXED,1024,67.3,11.6,86.21 -minicpm4-0.5b,INT4-MIXED,32,40,11.9,84.03 -minicpm4-0.5b,INT8-CW,1024,74.7,12.2,81.97 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,45.1,12.6,79.37 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,70.6,12.8,78.13 -whisper-small,INT4-MIXED,prompt0,119.2,13.2,75.76 -whisper-large-v3-turbo,FP16,prompt0,277.1,13.3,75.19 -whisper-small,INT4-MIXED,prompt0,118.8,13.4,74.63 -whisper-large-v3-turbo,FP16,prompt1,342.8,13.5,74.07 -whisper-small,INT4-MIXED,prompt0,129.3,13.5,74.07 -gemma-3-270m,INT4-MIXED,32,40.8,13.7,72.99 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,49.1,13.7,72.99 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,71.8,13.8,72.46 -whisper-small,INT4-MIXED,prompt1,186.5,13.8,72.46 -whisper-small,INT4-MIXED,prompt1,178.3,14,71.43 -gemma-3-270m,INT4-MIXED,1024,57.8,14.1,70.92 -whisper-small,INT4-MIXED,prompt1,185.2,14.1,70.92 -whisper-small,INT8-CW,prompt1,198.8,14.6,68.49 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,48.9,14.7,68.03 -gemma-3-270m,FP16,1024,53.5,14.8,67.57 -gemma-3-270m,INT8-CW,32,47.5,14.8,67.57 -gemma-3-270m,FP16,32,39,14.9,67.11 -whisper-small,INT8-CW,prompt0,127.1,14.9,67.11 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,76.3,15.1,66.23 -gemma-3-270m,INT8-CW,1024,59.1,15.2,65.79 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,139.1,16.8,59.52 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,63.7,17.1,58.48 -llama-3.2-1b-instruct,INT4-MIXED,32,41.8,17.2,58.14 -whisper-small,FP16,prompt0,128.2,17.2,58.14 -whisper-small,FP16,prompt1,174.1,17.2,58.14 -llama-3.2-1b-instruct,INT4-MIXED,1024,145.4,18.1,55.25 -qwen2.5-coder-0.5b-instruct,INT8-CW,32,64.7,18.1,55.25 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,68.9,18.3,54.64 -qwen3.5-0.8b,INT4-MIXED,1024,295.4,18.3,54.64 -qwen2.5-coder-0.5b-instruct,INT8-CW,1024,81.6,18.5,54.05 -qwen3.5-0.8b,INT4-MIXED,85,153.4,18.5,54.05 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,170.4,18.8,53.19 -qwen3.5-0.8b,INT4-MIXED,1024,298.1,18.9,52.91 -qwen3.5-0.8b,INT4-MIXED,85,166.2,19.3,51.81 -qwen3.5-0.8b,INT4-MIXED,85,178.4,19.9,50.25 -qwen3.5-0.8b,INT4-MIXED,1024,308.4,20,50.00 -deepseek-r1-distill-qwen-1.5b,INT8-CW,32,46.8,20.1,49.75 -qwen3.5-0.8b,INT8-CW,1024,320.6,20.6,48.54 -qwen3.5-0.8b,INT8-CW,85,193.3,20.6,48.54 -deepseek-r1-distill-qwen-1.5b,INT8-CW,1024,140.6,20.9,47.85 -nanollava,INT4-MIXED,760,170,21,47.62 -llama-3.2-1b-instruct,INT4-MIXED,32,38.8,21.2,47.17 -nanollava,INT8-CW,760,169.2,21.2,47.17 -llama-3.2-1b-instruct,INT4-MIXED,1024,120.7,21.5,46.51 -llama-3.2-1b-instruct,INT8-CW,32,39.1,21.5,46.51 -qwen2.5-coder-0.5b-instruct,FP16,32,38.9,21.7,46.08 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,78.7,22.4,44.64 -gemma-3-1b-it,INT4-MIXED,1024,127.7,22.5,44.44 -nanollava,INT8-CW,1752,281,22.5,44.44 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,161.3,22.5,44.44 -llama-3.2-1b-instruct,INT8-CW,1024,133,22.6,44.25 -minicpm4-0.5b,FP16,32,50.8,22.9,43.67 -nanollava,INT4-MIXED,1752,281.8,22.9,43.67 -qwen2.5-coder-0.5b-instruct,FP16,1024,88.1,23.2,43.10 -minicpm4-0.5b,FP16,1024,92.2,23.8,42.02 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,83.4,24.5,40.82 -gemma-3-1b-it,INT4-MIXED,1024,142.3,24.6,40.65 -glm-edge-1.5b-chat,INT4-MIXED,32,90.8,24.7,40.49 -gemma-3-1b-it,INT4-MIXED,32,71.5,24.8,40.32 -qwen2.5-1.5b-instruct,INT4-MIXED,32,73.5,24.9,40.16 -gemma-3-1b-it,INT4-MIXED,1024,148.6,25,40.00 -gemma-3-1b-it,INT4-MIXED,32,60.2,25.2,39.68 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,199,25.3,39.53 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,69.4,25.3,39.53 -gemma-3-1b-it,INT4-MIXED,32,81.4,25.4,39.37 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,195.5,25.5,39.22 -qwen2.5-1.5b-instruct,INT4-MIXED,32,75,25.8,38.76 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,170,26.6,37.59 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,166.6,26.9,37.17 -glm-edge-1.5b-chat,INT4-MIXED,1024,268.1,27.1,36.90 -qwen2.5-1.5b-instruct,INT8-CW,32,74.6,27.5,36.36 -qwen2.5-coder-1.5b-instruct,INT8-CW,32,76.3,27.9,35.84 -glm-edge-1.5b-chat,INT8-CW,32,94.3,28.6,34.97 -qwen2.5-coder-1.5b-instruct,INT8-CW,1024,180.9,28.8,34.72 -qwen2.5-1.5b-instruct,INT8-CW,1024,173.6,29.1,34.36 -gemma-3-1b-it,INT8-CW,1024,152,29.5,33.90 -glm-edge-1.5b-chat,INT8-CW,1024,244.1,30.1,33.22 -gemma-3-1b-it,INT8-CW,32,93.1,30.2,33.11 -nanollava,FP16,760,202.5,32.7,30.58 -nanollava,FP16,1752,324,33.8,29.59 -phi-2,INT4-MIXED,32,136.9,34.8,28.74 -llama-3.2-3b-instruct,INT4-MIXED,32,96,35.6,28.09 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,98.3,35.7,28.01 -stable-zephyr-3b-dpo,INT4-MIXED,32,126.5,36.5,27.40 -phi-2,INT4-MIXED,1024,393.7,36.6,27.32 -stablelm-3b-4e1t,INT4-MIXED,32,123.3,36.7,27.25 -afm-4.5b,INT4-MIXED,32,85.2,36.9,27.10 -llama-3.2-3b-instruct,INT4-MIXED,1024,326.2,37.2,26.88 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,295.6,37.3,26.81 -phi-2,INT4-MIXED,32,150.6,37.8,26.46 -llama-3.2-3b-instruct,INT4-MIXED,32,96.4,37.9,26.39 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,123.5,37.9,26.39 -qwen3.5-0.8b,FP16,85,230.3,38,26.32 -afm-4.5b,INT4-MIXED,1024,460.5,38.3,26.11 -stable-zephyr-3b-dpo,INT4-MIXED,1024,387.3,38.4,26.04 -phi-3-mini-128k-instruct,INT4-MIXED,32,112.8,38.5,25.97 -qwen3.5-0.8b,FP16,1024,397.7,38.6,25.91 -phi-3-mini-4k-instruct,INT4-MIXED,32,106.8,38.7,25.84 -phi-3.5-mini-instruct,INT4-MIXED,32,114.7,38.7,25.84 -stablelm-3b-4e1t,INT4-MIXED,1024,384.3,38.8,25.77 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,497.2,39.6,25.25 -stable-zephyr-3b-dpo,INT4-MIXED,32,140,39.7,25.19 -llama-3.2-3b-instruct,INT4-MIXED,1024,417.6,39.9,25.06 -phi-2,INT4-MIXED,1024,486.7,40.3,24.81 -chatglm3-6b,INT4-MIXED,32,73.5,40.6,24.63 -phi-3-mini-128k-instruct,INT4-MIXED,32,112.9,41.6,24.04 -phi-3-mini-128k-instruct,INT4-MIXED,1024,500.9,41.7,23.98 -phi-3-mini-4k-instruct,INT4-MIXED,1024,505.8,41.8,23.92 -deepseek-r1-distill-qwen-1.5b,FP16,32,66.6,41.9,23.87 -phi-3.5-mini-instruct,INT4-MIXED,1024,511.7,41.9,23.87 -stable-zephyr-3b-dpo,INT4-MIXED,1024,462.5,41.9,23.87 -chatglm3-6b,INT4-MIXED,1024,484.7,42.1,23.75 -gemma-3-1b-it,FP16,1024,178.3,42.4,23.58 -deepseek-r1-distill-qwen-1.5b,FP16,1024,200.9,42.5,23.53 -gemma-3-1b-it,FP16,32,88.6,42.5,23.53 -chatglm3-6b,INT4-MIXED,32,68.5,43,23.26 -phi-3-mini-128k-instruct,INT4-MIXED,1024,655,44.3,22.57 -chatglm3-6b,INT4-MIXED,1024,678.8,44.5,22.47 -llama-3.2-3b-instruct,INT4-MIXED,32,87,44.7,22.37 -phi-2,INT8-CW,32,117.5,44.7,22.37 -phi-3-mini-4k-instruct,INT4-MIXED,32,113.8,44.9,22.27 -llama-3.2-1b-instruct,FP16,32,62.9,45.1,22.17 -qwen3-4b,INT4-MIXED,32,130.7,45.1,22.17 -stablelm-3b-4e1t,INT4-MIXED,32,132.6,45.1,22.17 -phi-3.5-mini-instruct,INT4-MIXED,32,122.1,45.3,22.08 -phi-4-mini-reasoning,INT4-MIXED,32,147.1,45.4,22.03 -stablelm-3b-4e1t,INT8-CW,32,120.9,45.5,21.98 -phi-4-mini-instruct,INT4-MIXED,32,139.3,45.7,21.88 -stable-zephyr-3b-dpo,INT8-CW,32,130,45.8,21.83 -llama-3.2-1b-instruct,FP16,1024,202.3,45.9,21.79 -llama-3.2-3b-instruct,INT4-MIXED,1024,335.3,46.6,21.46 -phi-4-mini-reasoning,INT4-MIXED,1024,430.5,47.2,21.19 -falcon-7b-instruct,INT4-MIXED,32,87.8,47.8,20.92 -phi-4-mini-instruct,INT4-MIXED,1024,423.4,47.8,20.92 -biomistral-7b-slerp,INT4-MIXED,7,65.7,47.9,20.88 -phi-2,INT8-CW,1024,410.1,48.1,20.79 -phi-3-mini-4k-instruct,INT4-MIXED,1024,643.9,48.2,20.75 -phi-4-mini-reasoning,INT4-MIXED,32,156.7,48.2,20.75 -qwen3-4b,INT4-MIXED,1024,474.9,48.5,20.62 -stablelm-3b-4e1t,INT4-MIXED,1024,414.6,48.5,20.62 -internvl2-4b,INT4-MIXED,297,308.3,48.7,20.53 -phi-3.5-mini-instruct,INT4-MIXED,1024,667.8,48.8,20.49 -stable-zephyr-3b-dpo,INT8-CW,1024,413.9,48.8,20.49 -stablelm-3b-4e1t,INT8-CW,1024,422.7,49,20.41 -falcon-7b-instruct,INT4-MIXED,1024,596.1,49.4,20.24 -phi-4-mini-reasoning,INT4-MIXED,1024,533.3,49.7,20.12 -llama-3.2-3b-instruct,INT8-CW,32,85.7,50.5,19.80 -phi-3.5-vision-instruct,INT4-MIXED,802,695.8,50.6,19.76 -phi-3-mini-4k-instruct,INT4-MIXED,32,114.5,50.7,19.72 -qwen2.5-coder-3b-instruct,INT8-CW,32,128.4,50.8,19.69 -biomistral-7b-slerp,INT4-MIXED,7,74.8,51.1,19.57 -phi-4-mini-instruct,INT4-MIXED,32,161.3,51.1,19.57 -phi-3.5-mini-instruct,INT4-MIXED,32,110.4,51.3,19.49 -afm-4.5b,INT8-CW,32,76.3,51.5,19.42 -qwen2.5-coder-3b-instruct,INT8-CW,1024,412.6,52.1,19.19 -llama-3.2-3b-instruct,INT8-CW,1024,355.7,52.3,19.12 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,84.5,52.5,19.05 -internvl2-4b,INT4-MIXED,1027,814.3,52.7,18.98 -phi-3.5-vision-instruct,INT4-MIXED,1032,863.8,52.7,18.98 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,68.3,52.8,18.94 -afm-4.5b,INT8-CW,1024,344.2,53,18.87 -deepseek-r1-distill-qwen-7b,INT4-MIXED,32,80.4,53.8,18.59 -phi-4-mini-instruct,INT4-MIXED,1024,565.7,54,18.52 -phi-4-mini-reasoning,INT4-MIXED,32,162.2,54.4,18.38 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,315.5,54.4,18.38 -phi-3.5-mini-instruct,INT4-MIXED,1024,508.3,54.5,18.35 -phi-3-mini-4k-instruct,INT4-MIXED,1024,502.7,54.6,18.32 -deepseek-r1-distill-llama-8b,INT4-MIXED,1024,580.6,54.7,18.28 -phi-4-mini-instruct,INT4-MIXED,32,130.5,55,18.18 -deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,724.8,55.3,18.08 -whisper-large-v3,INT4-MIXED,prompt1,550.9,55.3,18.08 -whisper-large-v3,INT4-MIXED,prompt0,502.8,55.7,17.95 -whisper-large-v3,INT8-CW,prompt1,526.1,55.7,17.95 -gemma-4-e2b-it,INT4-MIXED,274,861.1,55.8,17.92 -falcon-7b-instruct,INT4-MIXED,32,91.6,56,17.86 -internvl2-4b,INT4-MIXED,297,275.5,56.2,17.79 -qwen2.5-1.5b-instruct,FP16,32,89.8,56.9,17.57 -qwen2.5-coder-1.5b-instruct,FP16,32,91.6,56.9,17.57 -phi-4-mini-instruct,INT4-MIXED,1024,431.9,57.4,17.42 -phi-4-mini-reasoning,INT4-MIXED,1024,442.9,57.4,17.42 -falcon-7b-instruct,INT4-MIXED,1024,795.7,57.5,17.39 -qwen2.5-coder-1.5b-instruct,FP16,1024,254.6,57.6,17.36 -glm-edge-4b-chat,INT4-MIXED,32,206,57.7,17.33 -qwen2.5-1.5b-instruct,FP16,1024,249.4,57.7,17.33 -whisper-large-v3,INT8-CW,prompt0,473.8,57.7,17.33 -phi-3-mini-4k-instruct,INT8-CW,32,105.6,57.9,17.27 -phi-3-mini-128k-instruct,INT8-CW,32,102.7,58,17.24 -phi-3.5-mini-instruct,INT8-CW,32,96.8,58.1,17.21 -glm-edge-1.5b-chat,FP16,32,131.8,58.3,17.15 -llama-2-7b-chat-hf,INT4-MIXED,32,118.9,58.3,17.15 -glm-edge-1.5b-chat,FP16,1024,315.8,59.2,16.89 -qwen3-4b,INT4-MIXED,32,124.1,59.4,16.84 -glm-edge-4b-chat,INT4-MIXED,1024,701.1,59.7,16.75 -deepseek-r1-distill-qwen-7b,INT4-MIXED,32,75.3,60,16.67 -gemma-4-e2b-it,INT4-MIXED,274,875.5,60.1,16.64 -internvl2-4b,INT4-MIXED,1027,713.8,61,16.39 -mistral-7b-instruct-v0.3,INT4-MIXED,32,106.4,61.1,16.37 -gemma-3-4b-it,INT4-MIXED,32,220.3,61.3,16.31 -mistral-7b-instruct-v0.2,INT4-MIXED,32,108.5,61.3,16.31 -deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,537.7,61.4,16.29 -phi-4-mini-instruct,INT8-CW,32,146.1,61.4,16.29 -phi-4-mini-reasoning,INT8-CW,32,121.7,61.4,16.29 -phi-3-mini-128k-instruct,INT8-CW,1024,541.5,61.5,16.26 -phi-3.5-mini-instruct,INT8-CW,1024,543.9,61.7,16.21 -phi-3-mini-4k-instruct,INT8-CW,1024,543,61.8,16.18 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,86.2,62.6,15.97 -gemma-4-e2b-it,INT4-MIXED,1024,1052.7,62.7,15.95 -gemma-3-4b-it,INT4-MIXED,32,213.2,62.9,15.90 -internvl2-4b,INT8-CW,297,299.9,63.1,15.85 -qwen3-4b,INT4-MIXED,1024,480.7,63.1,15.85 -gemma-3-4b-it,INT4-MIXED,1024,513,63.3,15.80 -llama-2-7b-chat-hf,INT4-MIXED,1024,715.1,63.4,15.77 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,733.2,63.8,15.67 -phi-4-mini-instruct,INT8-CW,1024,464.7,63.8,15.67 -phi-4-mini-reasoning,INT8-CW,1024,465.8,63.8,15.67 -qwen3-vl-4b-instruct,INT4-MIXED,4907,7718.6,64,15.63 -qwen3-4b,INT8-CW,32,115.3,64.1,15.60 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,731.2,64.2,15.58 -qwen3-vl-4b-instruct,INT4-MIXED,4937,7877.4,64.2,15.58 -qwen2.5-7b-instruct-1m,INT4-MIXED,32,93.3,64.3,15.55 -qwen2.5-7b-instruct,INT4-MIXED,32,89.6,64.4,15.53 -mistral-7b-instruct-v0.2,INT4-MIXED,32,130.3,64.5,15.50 -mistral-7b-instruct-v0.3,INT4-MIXED,32,110.3,64.6,15.48 -deepseek-r1-distill-llama-8b,INT4-MIXED,1024,600,64.7,15.46 -whisper-large-v3,FP16,prompt0,510.4,64.8,15.43 -mistral-7b-instruct-v0.1,INT4-MIXED,32,113.8,65,15.38 -whisper-large-v3,FP16,prompt1,566.9,65,15.38 -gemma-4-e2b-it,INT4-MIXED,1024,1082.6,65.1,15.36 -gemma-4-e2b-it,INT8-CW,274,844.5,65.1,15.36 -qwen2.5-7b-instruct,INT4-MIXED,1024,645.4,65.4,15.29 -qwen2.5-7b-instruct-1m,INT4-MIXED,1024,654.6,65.4,15.29 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,99.8,65.5,15.27 -gemma-3-4b-it,INT4-MIXED,1024,617.2,66.2,15.11 -gpt-oss-20b,INT4-MIXED,32,250.6,66.2,15.11 -llama-3.1-8b-instruct,INT4-MIXED,32,118.3,66.3,15.08 -phi-3.5-vision-instruct,INT8-CW,802,602.3,66.3,15.08 -llama-3-8b-instruct,INT4-MIXED,32,110.7,66.5,15.04 -qwen3-4b,INT8-CW,1024,502.4,66.9,14.95 -gpt-oss-20b,INT4-MIXED,1024,3204.9,67.2,14.88 -mistral-7b-instruct-v0.1,INT4-MIXED,1025,1008.6,67.4,14.84 -deepseek-r1-distill-llama-8b,INT4-MIXED,1024,759.5,67.6,14.79 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,999.4,67.6,14.79 -phi-3.5-vision-instruct,INT8-CW,1032,775.8,67.6,14.79 -qwen3-vl-4b-instruct,INT4-MIXED,4937,8299.5,67.6,14.79 -internvl2-4b,INT8-CW,1027,732.4,67.7,14.77 -minicpm-v-2_6,INT4-MIXED,228,726.5,67.7,14.77 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,1022.7,67.7,14.77 -minicpm-o-2_6,INT4-MIXED,238,744,68,14.71 -chatglm3-6b,INT8-CW,32,90.8,68.1,14.68 -minicpm4-8b,INT4-MIXED,32,120.3,68.1,14.68 -qwen2.5-7b-instruct,INT4-MIXED,32,119.1,68.1,14.68 -qwen3-vl-4b-instruct,INT4-MIXED,4907,8220.5,68.1,14.68 -qwen2-7b-instruct,INT4-MIXED,32,114.5,68.2,14.66 -qwen2.5-7b-instruct-1m,INT4-MIXED,32,109.2,68.3,14.64 -minicpm3-4b,INT4-MIXED,32,309.7,68.5,14.60 -glm-edge-4b-chat,INT8-CW,32,192.4,68.9,14.51 -phi-4-multimodal-instruct,INT4-MIXED,578,598.2,68.9,14.51 -phi-4-multimodal-instruct,INT4-MIXED,786,685.6,68.9,14.51 -llama-3-8b-instruct,INT4-MIXED,1024,714.2,69.4,14.41 -qwen3-8b,INT4-MIXED,32,135.8,69.6,14.37 -chatglm3-6b,INT8-CW,1024,481.5,69.7,14.35 -llama-3.1-8b-instruct,INT4-MIXED,1024,729,69.7,14.35 -qwen2.5-7b-instruct,INT4-MIXED,1024,897.8,69.8,14.33 -phi-4-multimodal-instruct,INT4-MIXED,1362,1481.7,70.2,14.25 -qwen2.5-7b-instruct-1m,INT4-MIXED,1024,894,70.2,14.25 -minicpm4-8b,INT4-MIXED,1024,798.7,70.4,14.20 -phi-4-multimodal-instruct,INT4-MIXED,1570,1638.5,70.4,14.20 -qwen2-7b-instruct,INT4-MIXED,1024,889.5,70.4,14.20 -llama-3-8b-instruct,INT4-MIXED,32,121.2,70.5,14.18 -gemma-3-4b-it,INT4-MIXED,32,208.8,71,14.08 -gemma-4-e2b-it,INT8-CW,1024,1063.7,71.7,13.95 -glm-edge-4b-chat,INT8-CW,1024,613.1,71.9,13.91 -minicpm-v-2_6,INT4-MIXED,228,811.4,72,13.89 -minicpm-o-2_6,INT4-MIXED,238,806.5,72.2,13.85 -qwen3-8b,INT4-MIXED,1024,775.9,72.3,13.83 -minicpm4-8b,INT4-MIXED,32,136.6,72.6,13.77 -qwen3-8b,INT4-MIXED,32,139.6,73,13.70 -llama-3-8b-instruct,INT4-MIXED,1024,981.8,73.1,13.68 -minicpm-v-4_5,INT4-MIXED,217,830.3,73.9,13.53 -gemma-3-4b-it,INT8-CW,32,205.4,74.4,13.44 -minicpm4-8b,INT4-MIXED,1024,1088.7,74.5,13.42 -minicpm3-4b,INT4-MIXED,32,355.7,75.5,13.25 -gemma-3-4b-it,INT4-MIXED,1024,523.7,76.5,13.07 -minicpm3-4b,INT4-MIXED,1024,830.4,76.9,13.00 -minicpm-v-4_5,INT4-MIXED,217,878.1,77.5,12.90 -minicpm3-4b,INT4-MIXED,32,295,77.5,12.90 -glm-4-9b-chat-hf,INT4-MIXED,32,130,77.6,12.89 -qwen3.5-9b,INT4-MIXED,83,458.9,78.3,12.77 -qwen3.5-9b,INT4-MIXED,1024,1267.7,78.8,12.69 -qwen3-vl-4b-instruct,INT4-MIXED,4937,7956,78.9,12.67 -gemma-3-4b-it,INT8-CW,1024,536.9,79,12.66 -gemma-7b-it,INT4-MIXED,32,118,79,12.66 -qwen3-vl-4b-instruct,INT4-MIXED,4907,7691.1,79.3,12.61 -minicpm3-4b,INT8-CW,32,328.7,80.2,12.47 -gpt-oss-20b,INT4-MIXED,32,289.1,80.3,12.45 -biomistral-7b-slerp,INT8-CW,7,107.1,80.5,12.42 -glm-4-9b-chat-hf,INT4-MIXED,1024,890.3,80.5,12.42 -deepseek-r1-distill-qwen-7b,INT8-CW,32,103.2,81.2,12.32 -llama-2-7b-chat-hf,INT4-MIXED,32,101.8,82.1,12.18 -qwen3.5-9b,INT4-MIXED,83,497.7,82.1,12.18 -glm-4-9b-chat-hf,INT4-MIXED,32,140.3,82.5,12.12 -qwen3.5-9b,INT4-MIXED,1024,1532,82.6,12.11 -phi-4-multimodal-instruct,INT8-CW,578,627.9,82.8,12.08 -qwen2.5-vl-7b-instruct,INT4-MIXED,32,314.1,82.8,12.08 -deepseek-r1-distill-qwen-7b,INT8-CW,1024,509.4,82.9,12.06 -qwen3-vl-4b-instruct,INT8-CW,4907,7760.6,83.1,12.03 -qwen3-vl-4b-instruct,INT8-CW,4937,7793.8,83.1,12.03 -llama-3.1-8b-instruct,INT4-MIXED,32,129.7,83.2,12.02 -gemma-7b-it,INT4-MIXED,1024,877,83.3,12.00 -phi-4-multimodal-instruct,INT8-CW,786,739.4,83.4,11.99 -qwen2-7b-instruct,INT4-MIXED,32,98.6,83.5,11.98 -minicpm3-4b,INT4-MIXED,1024,950.8,83.6,11.96 -qwen2.5-7b-instruct,INT4-MIXED,32,98.2,83.8,11.93 -gemma-7b-it,INT4-MIXED,32,128.3,83.9,11.92 -mistral-7b-instruct-v0.3,INT4-MIXED,32,104.4,83.9,11.92 -mistral-7b-instruct-v0.2,INT4-MIXED,32,102,84,11.90 -phi-4-multimodal-instruct,INT8-CW,1362,1565,84.1,11.89 -phi-4-multimodal-instruct,INT8-CW,1570,1759.2,84.1,11.89 -glm-4-9b-chat-hf,INT4-MIXED,1024,1176.7,84.2,11.88 -qwen2.5-vl-7b-instruct,INT4-MIXED,1024,838.1,84.5,11.83 -gpt-oss-20b,INT4-MIXED,1024,3427.6,85.2,11.74 -qwen2.5-7b-instruct,INT4-MIXED,1024,656.2,85.3,11.72 -minicpm3-4b,INT4-MIXED,1024,878,85.5,11.70 -qwen2-7b-instruct,INT4-MIXED,1024,655.1,85.5,11.70 -deepseek-r1-distill-llama-8b,INT8-CW,32,112.4,85.7,11.67 -llama-3.1-8b-instruct,INT4-MIXED,1024,947.9,85.9,11.64 -llava-next-video-7b-hf,INT4-MIXED,2945,4195.8,86.1,11.61 -phi-4-multimodal-instruct,INT4-MIXED,578,750.6,86.2,11.60 -minicpm-v-2_6,INT4-MIXED,228,705,86.4,11.57 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,700.9,86.4,11.57 -qwen2.5-vl-7b-instruct,INT4-MIXED,32,360.6,86.5,11.56 -fara-7b,INT4-MIXED,32,411.5,86.6,11.55 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,718,86.6,11.55 -stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,86,86.8,11.52 -llama-2-7b-chat-hf,INT4-MIXED,1024,717.3,86.9,11.51 -phi-4-multimodal-instruct,INT4-MIXED,1362,1827.9,87,11.49 -gemma-2-9b-it,INT4-MIXED,32,178.7,87.4,11.44 -minicpm3-4b,INT8-CW,1024,895.7,87.4,11.44 -deepseek-r1-distill-llama-8b,INT8-CW,1024,567.3,88,11.36 -phi-4-multimodal-instruct,INT4-MIXED,1570,2034.6,88,11.36 -phi-4-multimodal-instruct,INT4-MIXED,786,905.4,88.1,11.35 -gemma-7b-it,INT4-MIXED,1024,1156.8,88.2,11.34 -qwen2.5-vl-7b-instruct,INT4-MIXED,1024,1088.4,88.5,11.30 -fara-7b,INT4-MIXED,1024,1293.2,88.9,11.25 -llama-3-8b-instruct,INT4-MIXED,32,113,89.4,11.19 -llama-3-8b-instruct,INT4-MIXED,32,109.8,89.6,11.16 -llama-3.1-8b-instruct,INT4-MIXED,32,115.1,89.7,11.15 -phi-2,FP16,32,135.2,90,11.11 -stable-zephyr-3b-dpo,FP16,32,123.7,91.5,10.93 -stablelm-3b-4e1t,FP16,32,147.6,92,10.87 -llama-3-8b-instruct,INT4-MIXED,1024,725.3,92.1,10.86 -llama-3-8b-instruct,INT4-MIXED,1024,723.5,92.2,10.85 -llama-3.1-8b-instruct,INT4-MIXED,1024,714.3,92.2,10.85 -gemma-2-9b-it,INT4-MIXED,1024,1006.2,92.4,10.82 -qwen3-8b,INT4-MIXED,32,135.3,92.5,10.81 -stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,92.5,92.7,10.79 -minicpm4-8b,INT4-MIXED,32,116.4,93.3,10.72 -phi-2,FP16,1024,616.7,94.5,10.58 -minicpm4-8b,INT4-MIXED,1024,798.7,95.3,10.49 -qwen3-8b,INT4-MIXED,1024,774.1,95.8,10.44 -stable-zephyr-3b-dpo,FP16,1024,615.1,96.8,10.33 -stablelm-3b-4e1t,FP16,1024,598.5,96.8,10.33 -deepseek-r1-distill-qwen-14b,INT4-MIXED,32,146,97.3,10.28 -ltx-video,INT8-CW,11,98.3,97.9,10.21 -llama-2-7b-chat-hf,INT8-CW,32,139.2,98.4,10.16 \ No newline at end of file +distil-large-v2,INT4-MIXED,32,248.2,8.7,114.9425287 +distil-large-v2,INT8-CW,1024,284.5,8.8,113.6363636 +distil-large-v2,INT4-MIXED,1024,296.5,8.9,112.3595506 +distil-large-v2,INT8-CW,32,230.5,9,111.1111111 +minicpm4-0.5b,INT4-MIXED,32,35.6,9.1,109.8901099 +whisper-large-v3-turbo,INT8-CW,32,242.7,9.3,107.5268817 +whisper-large-v3-turbo,INT8-CW,1024,286.8,9.4,106.3829787 +whisper-large-v3-turbo,INT4-MIXED,1024,309.5,9.5,105.2631579 +whisper-large-v3-turbo,INT4-MIXED,32,262.9,9.6,104.1666667 +minicpm4-0.5b,INT4-MIXED,1024,60.6,10,100 +minicpm4-0.5b,INT4-MIXED,32,33.8,10,100 +minicpm4-0.5b,INT4-MIXED,32,36.8,10.4,96.15384615 +minicpm4-0.5b,INT4-MIXED,1024,70.8,11,90.90909091 +minicpm4-0.5b,INT8-CW,32,38,11.1,90.09009009 +minicpm4-0.5b,INT4-MIXED,1024,66,11.3,88.49557522 +nuextract-tiny,INT4-MIXED,32,32.7,11.3,88.49557522 +gemma-3-270m,INT4-MIXED,32,39.6,11.7,85.47008547 +nuextract-tiny,INT8-CW,32,44.7,11.8,84.74576271 +gemma-3-270m,INT4-MIXED,1024,53.4,12.1,82.6446281 +nuextract-tiny,INT4-MIXED,1024,73.1,12.1,82.6446281 +lfm2.5-350m,INT4-MIXED,32,45.2,12.4,80.64516129 +minicpm4-0.5b,INT8-CW,1024,65.5,12.4,80.64516129 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,39.3,12.5,80 +distil-large-v2,FP16,32,263.7,12.7,78.74015748 +nuextract-tiny,INT8-CW,1024,70.3,12.7,78.74015748 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,71.7,12.7,78.74015748 +distil-large-v2,FP16,1024,320.6,12.8,78.125 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,45.6,13,76.92307692 +whisper-large-v3-turbo,FP16,32,285,13.1,76.33587786 +whisper-small,INT4-MIXED,32,128.8,13.2,75.75757576 +whisper-small,INT4-MIXED,1024,179.5,13.2,75.75757576 +gemma-3-270m,INT8-CW,32,37.8,13.3,75.18796992 +whisper-large-v3-turbo,FP16,1024,346.3,13.3,75.18796992 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,76.2,13.5,74.07407407 +gemma-3-270m,INT8-CW,1024,53.2,13.7,72.99270073 +lfm2.5-350m,INT8-CW,32,41.4,13.7,72.99270073 +lfm2.5-350m,INT4-MIXED,1024,90.4,14.1,70.92198582 +lfm2.5-350m,INT8-CW,1024,90.8,14.1,70.92198582 +whisper-small,INT8-CW,32,128.7,14.1,70.92198582 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,47.7,14.2,70.42253521 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,67.7,14.7,68.02721088 +whisper-small,INT8-CW,1024,185.8,14.9,67.11409396 +gemma-3-270m,FP16,1024,50.9,15,66.66666667 +gemma-3-270m,FP16,32,37.5,15.1,66.22516556 +whisper-small,INT4-MIXED,1024,203.3,15.2,65.78947368 +minicpm5-1b,INT4-MIXED,32,46.8,15.6,64.1025641 +whisper-small,INT4-MIXED,32,125,15.7,63.69426752 +tiny-llama-1.1b-chat,INT4-MIXED,32,49.5,16.1,62.11180124 +whisper-small,INT4-MIXED,32,133.5,16.5,60.60606061 +llama-3.2-1b-instruct,INT4-MIXED,32,59.4,16.6,60.24096386 +tiny-llama-1.1b-chat,INT4-MIXED,1024,141.9,16.6,60.24096386 +minicpm5-1b,INT4-MIXED,1024,120,16.7,59.88023952 +minicpm5-1b,INT8-CW,32,48.3,16.8,59.52380952 +whisper-small,FP16,1024,183.8,17,58.82352941 +whisper-small,FP16,32,128.1,17.3,57.80346821 +whisper-small,INT4-MIXED,1024,193.2,17.3,57.80346821 +tiny-llama-1.1b-chat,INT4-MIXED,32,36,17.5,57.14285714 +qwen2.5-coder-0.5b-instruct,INT8-CW,32,62.8,18,55.55555556 +tiny-llama-1.1b-chat,INT4-MIXED,1024,119.3,18.1,55.24861878 +llama-3.2-1b-instruct,INT4-MIXED,1024,130.2,18.2,54.94505495 +minicpm5-1b,INT8-CW,1024,106.1,18.4,54.34782609 +qwen2.5-coder-0.5b-instruct,INT8-CW,1024,81.6,18.4,54.34782609 +qwen3.5-0.8b,INT4-MIXED,85,110.7,18.5,54.05405405 +qwen3.5-0.8b,INT4-MIXED,1024,241.3,18.7,53.47593583 +tiny-llama-1.1b-chat,INT8-CW,32,42,18.8,53.19148936 +qwen3.5-0.8b,INT4-MIXED,85,103.5,18.9,52.91005291 +qwen3.5-0.8b,INT4-MIXED,85,131.1,19.3,51.8134715 +nanollava,INT4-MIXED,760,150,19.4,51.54639175 +qwen3.5-0.8b,INT4-MIXED,1024,234.1,19.6,51.02040816 +tiny-llama-1.1b-chat,INT8-CW,1024,122.6,19.7,50.76142132 +qwen3.5-0.8b,INT4-MIXED,1024,256.9,19.8,50.50505051 +minicpm4-0.5b,FP16,32,42.2,20,50 +qwen3.5-0.8b,INT8-CW,85,124.4,20,50 +nanollava,INT8-CW,760,157.2,20.1,49.75124378 +qwen3.5-0.8b,INT8-CW,1024,251.6,20.5,48.7804878 +llama-3.2-1b-instruct,INT4-MIXED,32,38.4,20.8,48.07692308 +exaone-4.0-1.2b,INT4-MIXED,32,66.3,21.1,47.39336493 +llama-3.2-1b-instruct,INT8-CW,32,38,21.1,47.39336493 +llama-3.2-1b-instruct,INT4-MIXED,1024,112.5,21.3,46.94835681 +minicpm4-0.5b,FP16,1024,88.3,21.4,46.72897196 +nanollava,INT8-CW,1752,267.3,21.7,46.08294931 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,81.5,21.7,46.08294931 +lfm2.5-1.2b-instruct,INT4-MIXED,32,75.3,21.8,45.87155963 +gemma-3-1b-it,INT4-MIXED,1024,118.1,22,45.45454545 +gemma-3-1b-it,INT4-MIXED,32,52.2,22.3,44.84304933 +lfm2.5-350m,FP16,32,54.7,22.3,44.84304933 +llama-3.2-1b-instruct,INT8-CW,1024,129.8,22.4,44.64285714 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,148.9,22.4,44.64285714 +exaone-4.0-1.2b,INT4-MIXED,1024,202.4,22.5,44.44444444 +lfm2-1.2b,INT4-MIXED,32,72.2,22.5,44.44444444 +nanollava,INT4-MIXED,1752,279,22.6,44.24778761 +exaone-4.0-1.2b,INT4-MIXED,32,83.5,22.9,43.66812227 +lfm2-1.2b,INT4-MIXED,1024,165.2,22.9,43.66812227 +gemma-3-1b-it,INT4-MIXED,32,74.7,23,43.47826087 +gemma-3-1b-it,INT4-MIXED,32,79.1,23.1,43.29004329 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,166.3,23.1,43.29004329 +gemma-3-1b-it,INT4-MIXED,1024,134.5,23.3,42.91845494 +lfm2.5-350m,FP16,1024,98.2,23.4,42.73504274 +gemma-3-1b-it,INT4-MIXED,1024,121.6,23.5,42.55319149 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,89.6,23.8,42.01680672 +lfm2.5-1.2b-instruct,INT4-MIXED,32,79.6,24,41.66666667 +nuextract-tiny,FP16,32,57.3,24,41.66666667 +lfm2-1.2b,INT4-MIXED,32,80.5,24.2,41.32231405 +qwen2.5-coder-0.5b-instruct,FP16,32,70.2,24.4,40.98360656 +qwen2.5-1.5b-instruct,INT4-MIXED,32,87.6,24.6,40.6504065 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,72.9,24.6,40.6504065 +exaone-4.0-1.2b,INT8-CW,32,75.6,24.7,40.48582996 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,204.2,24.8,40.32258065 +lfm2-1.2b,INT4-MIXED,1024,196.5,24.9,40.16064257 +exaone-4.0-1.2b,INT4-MIXED,1024,220.3,25,40 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,156.2,25.1,39.84063745 +qwen2.5-1.5b-instruct,INT4-MIXED,32,68.8,25.1,39.84063745 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,187.7,25.1,39.84063745 +qwen2.5-coder-0.5b-instruct,FP16,1024,86.5,25.2,39.68253968 +nuextract-tiny,FP16,1024,88.5,25.3,39.5256917 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,156.1,25.4,39.37007874 +minicpm-1b-sft,INT4-MIXED,31,102.8,25.5,39.21568627 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,182.9,25.5,39.21568627 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,59.5,25.6,39.0625 +exaone-4.0-1.2b,INT8-CW,1024,203,26,38.46153846 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,84.5,26.1,38.31417625 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,157.7,26.1,38.31417625 +lfm2.5-1.2b-instruct,INT4-MIXED,32,59,26.1,38.31417625 +exaone-4.0-1.2b,INT4-MIXED,32,57.1,26.5,37.73584906 +lfm2-1.2b,INT4-MIXED,32,61.4,26.7,37.45318352 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,199.9,26.8,37.31343284 +minicpm-1b-sft,INT4-MIXED,31,113.4,26.9,37.17472119 +minicpm-1b-sft,INT4-MIXED,1014,174.9,27,37.03703704 +exaone-4.0-1.2b,INT4-MIXED,1024,201.2,27.1,36.900369 +deepseek-r1-distill-qwen-1.5b,INT8-CW,32,66.1,27.2,36.76470588 +lfm2-1.2b,INT4-MIXED,1024,175.8,27.2,36.76470588 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,173.8,27.2,36.76470588 +qwen2.5-coder-1.5b-instruct,INT8-CW,32,56,27.3,36.63003663 +qwen2.5-1.5b-instruct,INT8-CW,32,76,27.4,36.49635036 +gemma-3-1b-it,INT8-CW,32,102,27.9,35.84229391 +qwen2.5-1.5b-instruct,INT8-CW,1024,170.2,28,35.71428571 +lfm2.5-1.2b-instruct,INT8-CW,32,76,28.2,35.46099291 +lfm2-1.2b,INT8-CW,32,75.4,28.3,35.33568905 +qwen2.5-coder-1.5b-instruct,INT8-CW,1024,163.8,28.4,35.21126761 +minicpm-1b-sft,INT8-CW,31,89.9,28.5,35.0877193 +deepseek-r1-distill-qwen-1.5b,INT8-CW,1024,162,28.6,34.96503497 +minicpm-1b-sft,INT4-MIXED,31,114.9,28.6,34.96503497 +gemma-3-1b-it,INT8-CW,1024,135,29,34.48275862 +minicpm-1b-sft,INT4-MIXED,1014,211.8,29.4,34.01360544 +lfm2.5-1.2b-instruct,INT8-CW,1024,192.4,29.5,33.89830508 +lfm2-1.2b,INT8-CW,1024,187.3,29.7,33.67003367 +minicpm-1b-sft,INT8-CW,1014,185.1,30.8,32.46753247 +minicpm-1b-sft,INT4-MIXED,1014,180.9,31,32.25806452 +lfm2-8b-a1b,INT4-MIXED,32,167.5,31.9,31.34796238 +nanollava,FP16,760,191.6,32.2,31.05590062 +nanollava,FP16,1752,322.5,32.4,30.86419753 +lfm2-8b-a1b,INT4-MIXED,1024,1132.8,33.4,29.94011976 +phi-2,INT4-MIXED,32,131.3,34.5,28.98550725 +qwen3.5-0.8b,FP16,85,97.9,34.7,28.8184438 +smollm3-3b,INT4-MIXED,32,104.9,34.7,28.8184438 +smollm3-3b,INT4-MIXED,1024,287.7,35,28.57142857 +llama-3.2-3b-instruct,INT4-MIXED,32,90.2,35.3,28.3286119 +gemma-4-e2b-it,INT4-MIXED,274,461.1,35.4,28.24858757 +qwen3.5-0.8b,FP16,1024,269.5,35.6,28.08988764 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,105,35.7,28.01120448 +stable-zephyr-3b-dpo,INT4-MIXED,32,120.4,35.9,27.8551532 +minicpm5-1b,FP16,32,61.9,36.5,27.39726027 +phi-2,INT4-MIXED,1024,334,36.5,27.39726027 +gemma-4-e2b-it,INT4-MIXED,1024,627.2,36.7,27.2479564 +smollm3-3b,INT4-MIXED,32,118.1,36.7,27.2479564 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,276,36.8,27.17391304 +afm-4.5b,INT4-MIXED,32,84,37,27.02702703 +llama-3.2-3b-instruct,INT4-MIXED,1024,287,37,27.02702703 +tiny-llama-1.1b-chat,FP16,32,61.7,37.1,26.9541779 +stable-zephyr-3b-dpo,INT4-MIXED,1024,328.4,37.5,26.66666667 +phi-2,INT4-MIXED,32,144.9,37.6,26.59574468 +llama-3.2-3b-instruct,INT4-MIXED,32,100.6,37.8,26.45502646 +minicpm5-1b,FP16,1024,148.4,37.8,26.45502646 +smollm3-3b,INT4-MIXED,1024,373.7,37.9,26.38522427 +afm-4.5b,INT4-MIXED,1024,440.3,38,26.31578947 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,125.3,38.3,26.10966057 +tiny-llama-1.1b-chat,FP16,1024,173.3,38.5,25.97402597 +stable-zephyr-3b-dpo,INT4-MIXED,32,145,38.7,25.83979328 +llama-3.2-3b-instruct,INT4-MIXED,1024,389.8,39.3,25.44529262 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,465.1,39.6,25.25252525 +gemma-4-e2b-it,INT4-MIXED,274,485.8,40.1,24.93765586 +lfm2-8b-a1b,INT8-CW,32,180.8,40.4,24.75247525 +phi-2,INT4-MIXED,1024,427.4,40.4,24.75247525 +gemma-3-1b-it,FP16,32,67,40.8,24.50980392 +stable-zephyr-3b-dpo,INT4-MIXED,1024,408.5,41.5,24.09638554 +gemma-3-1b-it,FP16,1024,165.1,41.6,24.03846154 +lfm2-8b-a1b,INT8-CW,1024,932.4,41.7,23.98081535 +gemma-4-e2b-it,INT4-MIXED,1024,697.6,42,23.80952381 +gemma-4-e2b-it,INT8-CW,274,448,43.7,22.88329519 +phi-2,INT8-CW,32,113.7,43.9,22.77904328 +llama-3.2-3b-instruct,INT4-MIXED,32,88.4,44.6,22.42152466 +llama-3.2-1b-instruct,FP16,32,53.2,44.9,22.27171492 +qwen3-4b,INT4-MIXED,32,139.9,45,22.22222222 +phi-4-mini-reasoning,INT4-MIXED,32,141.9,45.2,22.12389381 +phi-4-mini-instruct,INT4-MIXED,32,137.9,45.4,22.02643172 +llama-3.2-1b-instruct,FP16,1024,197.9,45.6,21.92982456 +gemma-4-e2b-it,INT8-CW,1024,657.9,45.8,21.83406114 +llama-3.2-3b-instruct,INT4-MIXED,1024,309.3,46.1,21.69197397 +phi-4-mini-instruct,INT4-MIXED,1024,380.7,46.4,21.55172414 +phi-4-mini-reasoning,INT4-MIXED,1024,385.3,46.8,21.36752137 +phi-2,INT8-CW,1024,356.6,47.1,21.23142251 +qwen3-4b,INT4-MIXED,1024,410.4,47.4,21.09704641 +phi-4-mini-reasoning,INT4-MIXED,32,146.8,48.1,20.79002079 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,64.2,48.6,20.57613169 +phi-4-mini-reasoning,INT4-MIXED,1024,484,49.5,20.2020202 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,283.4,49.8,20.08032129 +phi-3.5-vision-instruct,INT4-MIXED,802,704,50.2,19.92031873 +smollm3-3b,INT8-CW,32,97.4,50.2,19.92031873 +llama-3.2-3b-instruct,INT8-CW,32,92.1,50.5,19.8019802 +qwen2.5-coder-3b-instruct,INT8-CW,32,113.1,50.5,19.8019802 +phi-4-mini-instruct,INT4-MIXED,32,145,50.7,19.72386588 +phi-4-mini-reasoning,INT4-MIXED,32,139.4,50.9,19.64636542 +phi-4-mini-instruct,INT4-MIXED,32,150.2,51.1,19.56947162 +phi-3.5-vision-instruct,INT4-MIXED,1032,844.7,51.5,19.41747573 +exaone-4.0-1.2b,FP16,32,104.9,51.6,19.37984496 +qwen2.5-coder-3b-instruct,INT8-CW,1024,385,51.8,19.30501931 +phi-4-mini-instruct,INT4-MIXED,1024,525.6,51.9,19.26782274 +smollm3-3b,INT8-CW,1024,308.5,51.9,19.26782274 +phi-4-mini-reasoning,INT4-MIXED,1024,405.1,52,19.23076923 +llama-3.2-3b-instruct,INT8-CW,1024,331.8,52.4,19.08396947 +phi-4-mini-instruct,INT4-MIXED,1024,401.4,52.6,19.01140684 +exaone-4.0-1.2b,FP16,1024,273.2,52.8,18.93939394 +smollm3-3b,INT4-MIXED,32,64.6,52.9,18.90359168 +lfm2.5-1.2b-instruct,FP16,32,93.4,53.4,18.72659176 +lfm2-1.2b,FP16,32,90.4,53.5,18.69158879 +whisper-large-v3,INT8-CW,32,458.1,53.6,18.65671642 +smollm3-3b,INT4-MIXED,1024,303.5,53.7,18.62197393 +lfm2.5-1.2b-instruct,FP16,1024,262.1,53.9,18.5528757 +lfm2-1.2b,FP16,1024,262.4,54,18.51851852 +whisper-large-v3,INT4-MIXED,32,473.2,54,18.51851852 +whisper-large-v3,INT8-CW,1024,525.1,54,18.51851852 +whisper-large-v3,INT4-MIXED,1024,538.4,54.3,18.41620626 +minicpm-1b-sft,FP16,31,145.8,54.4,18.38235294 +bloomz-7b1,INT4-MIXED,32,80.3,54.6,18.31501832 +minicpm-1b-sft,FP16,1014,256.9,55.6,17.98561151 +qwen2.5-coder-1.5b-instruct,FP16,32,89.6,56,17.85714286 +qwen3-4b,INT4-MIXED,32,131,56.4,17.73049645 +deepseek-r1-distill-qwen-1.5b,FP16,32,94,56.5,17.69911504 +qwen2.5-1.5b-instruct,FP16,32,100,56.5,17.69911504 +bloomz-7b1,INT4-MIXED,1024,887.3,57.1,17.51313485 +qwen2.5-coder-1.5b-instruct,FP16,1024,246.3,57.1,17.51313485 +qwen2.5-1.5b-instruct,FP16,1024,239,57.2,17.48251748 +codellama-7b,INT4-MIXED,32,124.9,58,17.24137931 +deepseek-r1-distill-qwen-1.5b,FP16,1024,236.6,58.1,17.21170396 +llama-2-7b-chat-hf,INT4-MIXED,32,117.3,58.1,17.21170396 +qwen3-4b,INT4-MIXED,1024,437.6,59.1,16.92047377 +lfm2-24b-a2b,INT4-MIXED,32,506.9,59.2,16.89189189 +gemma-3-4b-it,INT4-MIXED,32,199.1,59.4,16.83501684 +phi-4-mini-reasoning,INT8-CW,32,137.5,60.3,16.58374793 +phi-4-mini-instruct,INT8-CW,32,125.1,60.6,16.50165017 +codellama-7b,INT4-MIXED,1024,615.3,61,16.39344262 +zephyr-7b-beta,INT4-MIXED,32,105.1,61.2,16.33986928 +neural-chat-7b-v3-3,INT4-MIXED,32,103.4,61.3,16.3132137 +mistral-7b-instruct-v0.3,INT4-MIXED,32,104.2,61.5,16.2601626 +codellama-7b,INT4-MIXED,32,112.1,61.6,16.23376623 +biomistral-7b-slerp,INT4-MIXED,7,88,61.7,16.20745543 +mistral-7b-instruct-v0.2,INT4-MIXED,32,112.5,61.8,16.18122977 +llama-2-7b-chat-hf,INT4-MIXED,1024,637.8,62,16.12903226 +falcon-7b-instruct,INT4-MIXED,32,121.9,62.1,16.10305958 +gemma-3-4b-it,INT4-MIXED,32,187.1,62.5,16 +qwen3-vl-4b-instruct,INT4-MIXED,4907,7492.3,62.5,16 +qwen3-vl-4b-instruct,INT4-MIXED,4937,7660.2,62.6,15.97444089 +phi-4-mini-instruct,INT8-CW,1024,421,62.8,15.92356688 +phi-4-mini-reasoning,INT8-CW,1024,426.4,63.3,15.79778831 +zephyr-7b-beta,INT4-MIXED,1024,656.4,63.3,15.79778831 +neural-chat-7b-v3-3,INT4-MIXED,1024,662.7,63.4,15.77287066 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,662.6,63.5,15.7480315 +gemma-3-4b-it,INT4-MIXED,1024,513,63.7,15.69858713 +falcon-7b-instruct,INT4-MIXED,1024,684.6,63.9,15.64945227 +whisper-large-v3,FP16,32,490.7,63.9,15.64945227 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,664.5,64,15.625 +codellama-7b,INT4-MIXED,1024,880.2,64.1,15.60062402 +qwen3-4b,INT8-CW,32,117.9,64.1,15.60062402 +whisper-large-v3,FP16,1024,565,64.1,15.60062402 +qwen2.5-7b-instruct-1m,INT4-MIXED,32,107.5,64.2,15.57632399 +qwen2.5-7b-instruct,INT4-MIXED,32,87.6,64.4,15.52795031 +biomistral-7b-slerp,INT4-MIXED,7,98.9,64.7,15.45595054 +phi-3.5-vision-instruct,INT8-CW,802,591.1,64.7,15.45595054 +neural-chat-7b-v3-3,INT4-MIXED,32,122.1,64.8,15.43209877 +mistral-7b-instruct-v0.1,INT4-MIXED,32,116.5,64.9,15.40832049 +mistral-7b-instruct-v0.2,INT4-MIXED,32,97.1,64.9,15.40832049 +qwen2.5-7b-instruct,INT4-MIXED,1024,607.4,64.9,15.40832049 +gemma-3-4b-it,INT4-MIXED,1024,603.4,65.1,15.3609831 +mistral-7b-instruct-v0.3,INT4-MIXED,32,119.5,65.1,15.3609831 +qwen2.5-7b-instruct-1m,INT4-MIXED,1024,597,65.3,15.31393568 +bloomz-7b1,INT4-MIXED,32,109.1,65.4,15.29051988 +qwen3-vl-4b-instruct,INT4-MIXED,4907,7999.4,65.5,15.26717557 +qwen3-vl-4b-instruct,INT4-MIXED,4937,8165.1,65.6,15.24390244 +phi-3.5-vision-instruct,INT8-CW,1032,764,66.2,15.10574018 +minicpm3-4b,INT4-MIXED,32,311.9,66.3,15.08295626 +afm-4.5b,INT8-CW,32,94.7,66.4,15.06024096 +llama-3.1-8b-instruct,INT4-MIXED,32,114.6,66.6,15.01501502 +qwen3-4b,INT8-CW,1024,449.2,66.6,15.01501502 +llama-3-8b-instruct,INT4-MIXED,32,118.5,66.8,14.97005988 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,936.4,66.8,14.97005988 +deepseek-r1-distill-llama-8b,INT4-MIXED,32,124.7,66.9,14.94768311 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,934.2,66.9,14.94768311 +neural-chat-7b-v3-3,INT4-MIXED,1024,929.8,66.9,14.94768311 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,171.3,67.1,14.90312966 +minicpm-v-2_6,INT4-MIXED,228,718.1,67.3,14.85884101 +mistral-7b-instruct-v0.1,INT4-MIXED,1025,936.8,67.4,14.83679525 +gpt-oss-20b,INT4-MIXED,32,251.2,67.5,14.81481481 +phi-4-multimodal-instruct,INT4-MIXED,578,586.6,67.6,14.79289941 +phi-4-multimodal-instruct,INT4-MIXED,786,669.7,68,14.70588235 +qwen2.5-7b-instruct,INT4-MIXED,32,104.2,68.1,14.68428781 +qwen2-7b-instruct,INT4-MIXED,32,104.1,68.2,14.6627566 +qwen2.5-7b-instruct-1m,INT4-MIXED,32,111.6,68.2,14.6627566 +minicpm-o-2_6,INT4-MIXED,238,688,68.3,14.64128843 +afm-4.5b,INT8-CW,1024,419.9,68.5,14.59854015 +minicpm4-8b,INT4-MIXED,32,120.9,68.5,14.59854015 +deepseek-r1-distill-qwen-7b,INT4-MIXED,32,124.3,68.6,14.57725948 +gpt-oss-20b,INT4-MIXED,1024,3065.9,68.6,14.57725948 +bloomz-7b1,INT4-MIXED,1024,845.1,68.8,14.53488372 +llama-3.1-8b-instruct,INT4-MIXED,1024,666.3,68.8,14.53488372 +phi-4-multimodal-instruct,INT4-MIXED,1362,1405.8,69,14.49275362 +deepseek-r1-distill-llama-8b,INT4-MIXED,1024,664,69.1,14.47178003 +qwen3-8b,INT4-MIXED,32,151.4,69.1,14.47178003 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,711.9,69.3,14.43001443 +llama-3-8b-instruct,INT4-MIXED,1024,676.1,69.4,14.4092219 +qwen2-7b-instruct,INT4-MIXED,1024,859,69.6,14.36781609 +qwen2.5-7b-instruct,INT4-MIXED,1024,858.5,69.6,14.36781609 +qwen2.5-7b-instruct-1m,INT4-MIXED,1024,851.8,69.6,14.36781609 +phi-4-multimodal-instruct,INT4-MIXED,1570,1630.8,69.8,14.32664756 +deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,847.3,69.9,14.30615165 +gemma-3-4b-it,INT4-MIXED,32,171.9,70.4,14.20454545 +minicpm4-8b,INT4-MIXED,1024,737.8,70.6,14.16430595 +llama-3-8b-instruct,INT4-MIXED,32,123.1,70.8,14.12429379 +qwen3-8b,INT4-MIXED,1024,700.4,71.2,14.04494382 +falcon-7b-instruct,INT4-MIXED,32,119,71.7,13.94700139 +fara-7b,INT4-MIXED,32,266.1,71.7,13.94700139 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,179.5,71.9,13.90820584 +minicpm-v-2_6,INT4-MIXED,228,806.3,72.1,13.86962552 +llama-3-8b-instruct,INT4-MIXED,1024,925.4,72.5,13.79310345 +minicpm-o-2_6,INT4-MIXED,238,785.6,72.6,13.77410468 +minicpm3-4b,INT4-MIXED,32,354.5,72.6,13.77410468 +minicpm4-8b,INT4-MIXED,32,116.2,72.9,13.71742112 +qwen3-8b,INT4-MIXED,32,143.9,73,13.69863014 +minicpm3-4b,INT4-MIXED,1024,819.7,73.2,13.66120219 +minicpm3-4b,INT4-MIXED,32,315.8,73.3,13.6425648 +falcon-7b-instruct,INT4-MIXED,1024,924.6,73.4,13.6239782 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,963.4,73.6,13.58695652 +fara-7b,INT4-MIXED,1024,1167.1,73.9,13.53179973 +mistral-7b-instruct-v0.3,INT4-MIXED,32,112.9,73.9,13.53179973 +minicpm-v-4_5,INT4-MIXED,217,758.4,74.2,13.47708895 +gemma-3-4b-it,INT8-CW,32,191,74.3,13.4589502 +minicpm4-8b,INT4-MIXED,1024,1028.2,74.5,13.42281879 +qwen3-8b,INT4-MIXED,1024,960.8,75.1,13.31557923 +qwen3-vl-4b-instruct,INT4-MIXED,4907,7854.1,75.5,13.24503311 +qwen3-vl-4b-instruct,INT4-MIXED,4937,7886.6,75.5,13.24503311 +gemma-3-4b-it,INT4-MIXED,1024,511,75.6,13.22751323 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,674.2,76.2,13.12335958 +minicpm-v-4_5,INT4-MIXED,217,833.1,78.1,12.80409731 +qwen2-7b-instruct,INT4-MIXED,32,101,78.2,12.78772379 +qwen3.5-9b,INT4-MIXED,83,351,78.2,12.78772379 +gemma-7b-it,INT4-MIXED,32,127.3,78.3,12.77139208 +qwen3.5-9b,INT4-MIXED,1024,1122.8,78.5,12.7388535 +minicpm3-4b,INT8-CW,32,289.4,79,12.65822785 +videochat-flash-qwen2_5-7b_internvideo2-1b,INT4-MIXED,14531,61450.5,79.1,12.64222503 +minicpm3-4b,INT4-MIXED,1024,939.9,79.4,12.59445844 +gemma-3-4b-it,INT8-CW,1024,522.6,79.9,12.51564456 +qwen2-7b-instruct,INT4-MIXED,1024,617.2,79.9,12.51564456 +zephyr-7b-beta,INT4-MIXED,32,125.8,80,12.5 +qwen2.5-7b-instruct,INT4-MIXED,32,96.9,80.2,12.46882793 +minicpm3-4b,INT4-MIXED,1024,859.2,80.4,12.43781095 +gemma-7b-it,INT4-MIXED,1024,799.3,81.2,12.31527094 +qwen3.5-9b,INT4-MIXED,83,371.9,81.6,12.25490196 +llama-2-7b-chat-hf,INT4-MIXED,32,103.2,81.8,12.22493888 +qwen2.5-7b-instruct,INT4-MIXED,1024,622.3,81.9,12.21001221 +zephyr-7b-beta,INT4-MIXED,1024,692.9,81.9,12.21001221 +bloomz-7b1,INT8-CW,32,108,82,12.19512195 +phi-4-multimodal-instruct,INT8-CW,578,638,82.1,12.18026797 +phi-4-multimodal-instruct,INT8-CW,786,729.5,82.1,12.18026797 +qwen3.5-9b,INT4-MIXED,1024,1406.1,82.4,12.13592233 +deepseek-r1-distill-qwen-7b,INT4-MIXED,32,95.7,82.6,12.10653753 +phi-4-multimodal-instruct,INT8-CW,1362,1556.3,83.2,12.01923077 +gemma-7b-it,INT4-MIXED,32,124.6,83.4,11.99040767 +llama-3.1-8b-instruct,INT4-MIXED,32,125.6,83.4,11.99040767 +deepseek-r1-distill-llama-8b,INT4-MIXED,32,135.8,83.5,11.9760479 +phi-4-multimodal-instruct,INT8-CW,1570,1713.3,83.6,11.96172249 +deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,618.6,83.8,11.93317422 +gpt-oss-20b,INT4-MIXED,32,296.4,83.8,11.93317422 +mistral-7b-instruct-v0.2,INT4-MIXED,32,119.9,84,11.9047619 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,172,84.1,11.89060642 +llama-2-7b-chat-hf,INT4-MIXED,1024,633.2,84.4,11.84834123 +llama-3.1-8b-instruct,INT4-MIXED,1024,896.5,84.9,11.77856302 +deepseek-r1-distill-llama-8b,INT4-MIXED,1024,900.7,85,11.76470588 +phi-4-multimodal-instruct,INT4-MIXED,578,736.2,85,11.76470588 +phi-4-multimodal-instruct,INT4-MIXED,1362,1812.8,85.1,11.75088132 +bloomz-7b1,INT8-CW,1024,680.1,85.4,11.70960187 +minicpm-v-2_6,INT4-MIXED,228,713.1,85.4,11.70960187 +gemma-7b-it,INT4-MIXED,1024,1103.7,85.6,11.68224299 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,733.8,85.8,11.65501166 +phi-4-multimodal-instruct,INT4-MIXED,1570,2016.5,85.9,11.64144354 +phi-4-multimodal-instruct,INT4-MIXED,786,903.1,85.9,11.64144354 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,659.4,86,11.62790698 +minicpm3-4b,INT8-CW,1024,885,86.5,11.56069364 +gpt-oss-20b,INT4-MIXED,1024,3349.6,86.6,11.54734411 +gemma-2-9b-it,INT4-MIXED,32,167.3,87.7,11.40250855 +deepseek-r1-distill-llama-8b,INT4-MIXED,32,109.3,87.8,11.38952164 +phi-2,FP16,32,133.5,88,11.36363636 +llama-3.1-8b-instruct,INT4-MIXED,32,104.6,89.3,11.19820829 +stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,91.2,89.3,11.19820829 +deepseek-r1-distill-llama-8b,INT4-MIXED,1024,679.4,89.4,11.18568233 +llama-3-8b-instruct,INT4-MIXED,32,100.5,89.7,11.14827202 +qwen3-8b,INT4-MIXED,32,123.3,90.1,11.09877913 +llama-3-8b-instruct,INT4-MIXED,32,102.2,90.2,11.0864745 +gemma-2-9b-it,INT4-MIXED,1024,906.4,90.5,11.04972376 +llama-3.1-8b-instruct,INT4-MIXED,1024,670.5,91.6,10.91703057 +stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,95.3,91.6,10.91703057 +qwen3-8b,INT4-MIXED,1024,714,91.8,10.89324619 +llama-3-8b-instruct,INT4-MIXED,1024,672,91.9,10.88139282 +llama-3-8b-instruct,INT4-MIXED,1024,663.7,92.1,10.8577633 +minicpm4-8b,INT4-MIXED,32,109.6,92.5,10.81081081 +gemma-2-9b-it,INT4-MIXED,32,167,92.6,10.79913607 +phi-2,FP16,1024,541.4,92.8,10.77586207 +minicpm4-8b,INT4-MIXED,1024,762.6,94.3,10.60445387 +ltx-video,INT8-CW,11,98.9,95.8,10.43841336 +gemma-2-9b-it,INT4-MIXED,1024,1219.9,95.9,10.42752868 +codellama-7b,INT8-CW,32,123.1,96.1,10.40582726 +llama-2-7b-chat-hf,INT8-CW,32,130.2,98.2,10.18329939 +ltx-video,INT4-MIXED,11,93.2,98.6,10.14198783 +dolly-v2-12b,INT4-MIXED,32,173.1,98.9,10.11122346 \ No newline at end of file diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv index 898efb553194c3..e0ef7a57ae1951 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv +++ b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-258V.csv @@ -1,583 +1,559 @@ Topology,Precision,Input Size,1st latency (ms),2nd latency (ms),2nd token per sec -distil-large-v2,INT4-MIXED,prompt0,166.7,5.9,169.49 -distil-large-v2,INT4-MIXED,prompt1,211.5,5.9,169.49 -distil-large-v2,INT8-CW,prompt0,163.9,6,166.67 -distil-large-v2,INT8-CW,prompt1,206.9,6,166.67 -whisper-large-v3-turbo,INT8-CW,prompt1,225.5,6.3,158.73 -whisper-large-v3-turbo,INT8-CW,prompt0,184.1,6.9,144.93 -whisper-large-v3-turbo,INT4-MIXED,prompt0,183,7,142.86 -whisper-large-v3-turbo,INT4-MIXED,prompt1,232.7,7,142.86 -minicpm4-0.5b,INT4-MIXED,32,28.3,7.3,136.99 -minicpm4-0.5b,INT4-MIXED,1024,44.1,7.4,135.14 -minicpm4-0.5b,INT4-MIXED,32,28.4,7.4,135.14 -tiny-llama-1.1b-chat,INT4-MIXED,32,24.9,7.4,135.14 -minicpm4-0.5b,INT4-MIXED,1024,45.6,7.5,133.33 -minicpm4-0.5b,INT8-CW,32,30.5,7.6,131.58 -minicpm4-0.5b,INT4-MIXED,1024,53.1,7.7,129.87 -minicpm4-0.5b,INT4-MIXED,32,29.9,7.7,129.87 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,28.4,7.7,129.87 -gemma-3-270m,INT4-MIXED,1024,33.7,7.8,128.21 -minicpm4-0.5b,INT8-CW,1024,52.4,7.8,128.21 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,46.2,7.8,128.21 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,55,8,125.00 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,29.9,8,125.00 -tiny-llama-1.1b-chat,INT4-MIXED,32,26.4,8,125.00 -tiny-llama-1.1b-chat,INT4-MIXED,1024,87.4,8,125.00 -gemma-3-270m,INT4-MIXED,32,29.2,8.1,123.46 -distil-large-v2,FP16,prompt0,164.7,8.2,121.95 -distil-large-v2,FP16,prompt1,210,8.2,121.95 -gemma-3-270m,FP16,32,25.4,8.2,121.95 -gemma-3-270m,FP16,1024,32,8.3,120.48 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,28.7,8.5,117.65 -qwen2.5-coder-0.5b-instruct,INT8-CW,1024,53.2,8.6,116.28 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,45.7,8.7,114.94 -tiny-llama-1.1b-chat,INT4-MIXED,1024,112.5,8.7,114.94 -whisper-large-v3-turbo,FP16,prompt0,195.6,8.7,114.94 -gemma-3-270m,INT8-CW,1024,34.6,8.8,113.64 -qwen2.5-coder-0.5b-instruct,INT8-CW,32,30.8,8.8,113.64 -whisper-large-v3-turbo,FP16,prompt1,251.5,8.9,112.36 -gemma-3-270m,INT8-CW,32,29.1,9.2,108.70 -llama-3.2-1b-instruct,INT4-MIXED,32,21.5,9.5,105.26 -llama-3.2-1b-instruct,INT4-MIXED,32,22,9.6,104.17 -llama-3.2-1b-instruct,INT4-MIXED,1024,84,10,100.00 -llama-3.2-1b-instruct,INT4-MIXED,1024,98.2,10.2,98.04 -whisper-small,INT4-MIXED,prompt0,94.8,10.3,97.09 -whisper-small,INT4-MIXED,prompt1,137.9,10.5,95.24 -minicpm4-0.5b,FP16,32,24.8,10.9,91.74 -whisper-small,INT4-MIXED,prompt0,99.6,10.9,91.74 -whisper-small,INT4-MIXED,prompt1,145.1,10.9,91.74 -whisper-small,INT8-CW,prompt0,99.5,11,90.91 -whisper-small,INT8-CW,prompt1,143.3,11,90.91 -minicpm4-0.5b,FP16,1024,60.3,11.3,88.50 -qwen2.5-1.5b-instruct,INT4-MIXED,32,34.1,11.6,86.21 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,32.8,11.6,86.21 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,33.6,11.8,84.75 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,33.7,11.8,84.75 -qwen2.5-1.5b-instruct,INT4-MIXED,32,33.8,11.9,84.03 -gemma-3-1b-it,INT4-MIXED,1024,85.5,12,83.33 -gemma-3-1b-it,INT4-MIXED,1024,87.6,12,83.33 -nanollava,INT4-MIXED,760,98.8,12.1,82.64 -qwen2.5-coder-0.5b-instruct,FP16,32,27,12.1,82.64 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,36,12.1,82.64 -tiny-llama-1.1b-chat,INT8-CW,32,27.6,12.1,82.64 -whisper-small,FP16,prompt0,96.9,12.1,82.64 -whisper-small,FP16,prompt1,139.5,12.1,82.64 -nanollava,INT8-CW,760,94.7,12.2,81.97 -nanollava,INT4-MIXED,1752,190.2,12.3,81.30 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,135.5,12.3,81.30 -qwen2.5-coder-0.5b-instruct,FP16,1024,54,12.3,81.30 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,137.1,12.4,80.65 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,135.8,12.4,80.65 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,125.8,12.4,80.65 -nanollava,INT8-CW,1752,184.1,12.5,80.00 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,135.7,12.6,79.37 -qwen2.5-1.5b-instruct,INT4-MIXED,32,38.1,12.7,78.74 -tiny-llama-1.1b-chat,INT8-CW,1024,100.8,12.7,78.74 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,143.5,12.8,78.13 -gemma-3-1b-it,INT4-MIXED,1024,93.1,12.9,77.52 -qwen2.5-1.5b-instruct,INT4-MIXED,32,36.6,12.9,77.52 -nanollava,FP16,760,104.7,13,76.92 -gemma-3-1b-it,INT4-MIXED,32,38.6,13.1,76.34 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,144,13.3,75.19 -whisper-small,INT4-MIXED,prompt1,147.6,13.3,75.19 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,142.8,13.5,74.07 -gemma-3-1b-it,INT4-MIXED,32,37,13.6,73.53 -whisper-small,INT4-MIXED,prompt0,102.7,13.6,73.53 -gemma-3-1b-it,INT4-MIXED,32,39.5,14.1,70.92 -gemma-3-1b-it,INT8-CW,1024,99.1,14.3,69.93 -nanollava,FP16,1752,193,14.3,69.93 -llama-3.2-1b-instruct,INT8-CW,32,23.2,14.7,68.03 -gemma-3-1b-it,INT8-CW,32,43.5,14.9,67.11 -glm-edge-1.5b-chat,INT4-MIXED,1024,205,15,66.67 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,37.5,15.1,66.23 -glm-edge-1.5b-chat,INT4-MIXED,32,56.1,15.1,66.23 -llama-3.2-1b-instruct,INT8-CW,1024,101.5,15.5,64.52 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,155.6,15.9,62.89 -minicpm-1b-sft,INT4-MIXED,31,59.6,16.5,60.61 -minicpm-1b-sft,INT4-MIXED,1014,137.9,16.8,59.52 -minicpm-1b-sft,INT4-MIXED,31,60.3,17,58.82 -minicpm-1b-sft,INT4-MIXED,1014,161.3,17.1,58.48 -qwen3.5-0.8b,INT4-MIXED,85,147.3,17.3,57.80 -qwen3.5-0.8b,INT4-MIXED,1024,281,17.5,57.14 -gemma-2b-it,INT4-MIXED,32,26.9,17.8,56.18 -deepseek-r1-distill-qwen-1.5b,INT8-CW,32,38.3,17.9,55.87 -qwen3.5-0.8b,INT4-MIXED,1024,294.4,18,55.56 -qwen2.5-1.5b-instruct,INT8-CW,32,38,18.1,55.25 -minicpm-1b-sft,INT4-MIXED,1014,134,18.2,54.95 -minicpm-1b-sft,INT4-MIXED,31,60.6,18.2,54.95 -qwen2.5-1.5b-instruct,INT8-CW,32,38.3,18.2,54.95 -qwen2.5-coder-1.5b-instruct,INT8-CW,32,37.9,18.2,54.95 -qwen3.5-0.8b,INT4-MIXED,85,158,18.2,54.95 -glm-edge-1.5b-chat,INT8-CW,32,56.3,18.5,54.05 -qwen3.5-0.8b,INT8-CW,1024,296.7,18.5,54.05 -gemma-2b-it,INT4-MIXED,1024,171,18.6,53.76 -phi-2,INT4-MIXED,32,60.2,18.6,53.76 -gemma-2b-it,INT4-MIXED,32,32.2,18.7,53.48 -stable-zephyr-3b-dpo,INT4-MIXED,32,59.5,18.7,53.48 -stablelm-3b-4e1t,INT4-MIXED,32,59.6,18.7,53.48 -deepseek-r1-distill-qwen-1.5b,INT8-CW,1024,132.6,19,52.63 -minicpm-1b-sft,INT8-CW,31,66.5,19.1,52.36 -qwen2.5-1.5b-instruct,INT8-CW,1024,139.8,19.1,52.36 -qwen2.5-1.5b-instruct,INT8-CW,1024,145.4,19.1,52.36 -qwen2.5-coder-1.5b-instruct,INT8-CW,1024,141.9,19.1,52.36 -qwen3.5-0.8b,INT4-MIXED,1024,292.5,19.1,52.36 -qwen3.5-0.8b,INT8-CW,85,158.7,19.3,51.81 -gemma-2b-it,INT4-MIXED,1024,203.6,19.4,51.55 -glm-edge-1.5b-chat,INT8-CW,1024,211.3,19.7,50.76 -gemma-2-2b,INT4-MIXED,33,39,20,50.00 -phi-2,INT4-MIXED,32,63.3,20.2,49.50 -minicpm-1b-sft,INT8-CW,1014,150.6,20.4,49.02 -qwen3.5-0.8b,INT4-MIXED,85,150.1,20.5,48.78 -gemma-2-2b,INT4-MIXED,33,44.8,20.9,47.85 -phi-2,INT4-MIXED,1024,309.7,20.9,47.85 -stable-zephyr-3b-dpo,INT4-MIXED,1024,303.8,21,47.62 -stablelm-3b-4e1t,INT4-MIXED,1024,304.5,21,47.62 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,59.4,21.2,47.17 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,53,21.2,47.17 -gemma-2-2b,INT4-MIXED,1025,209.2,21.3,46.95 -qwen2.5-1.5b-instruct,INT4-MIXED,32,74.1,21.3,46.95 -stable-zephyr-3b-dpo,INT4-MIXED,32,67.9,21.3,46.95 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,185.2,21.4,46.73 -llama-3.2-3b-instruct,INT4-MIXED,32,43.9,21.7,46.08 -qwen2.5-1.5b-instruct,INT4-MIXED,32,75.8,22.1,45.25 -gemma-2-2b,INT4-MIXED,1025,227.1,22.2,45.05 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,193.9,22.2,45.05 -phi-2,INT4-MIXED,1024,383.8,22.5,44.44 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,234.8,22.5,44.44 -llama-3.2-3b-instruct,INT4-MIXED,32,43.9,22.6,44.25 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,58.4,22.6,44.25 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,240.1,22.6,44.25 -stablelm-3b-4e1t,INT4-MIXED,32,65.9,22.8,43.86 -llama-3.2-3b-instruct,INT4-MIXED,32,44.2,22.9,43.67 -llama-3.2-3b-instruct,INT4-MIXED,1024,243.1,23.3,42.92 -stable-zephyr-3b-dpo,INT4-MIXED,1024,336.7,23.5,42.55 -phi-3-mini-4k-instruct,INT4-MIXED,32,49.5,23.7,42.19 -gemma-3-1b-it,FP16,1024,108.2,23.8,42.02 -phi-3-mini-128k-instruct,INT4-MIXED,32,49,23.8,42.02 -phi-3.5-mini-instruct,INT4-MIXED,32,50,23.8,42.02 -qwen3.5-0.8b,FP16,1024,294,23.8,42.02 -qwen3.5-0.8b,FP16,85,122.2,23.8,42.02 -qwen2.5-coder-3b-instruct,INT4-MIXED,1024,427.9,24,41.67 -gemma-3-1b-it,FP16,32,40.5,24.1,41.49 -llama-3.2-3b-instruct,INT4-MIXED,1024,254,24.3,41.15 -llama-3.2-3b-instruct,INT4-MIXED,1024,277.5,24.6,40.65 -phi-3-mini-4k-instruct,INT4-MIXED,32,49.7,24.7,40.49 -phi-3.5-mini-instruct,INT4-MIXED,32,50.2,24.8,40.32 -stablelm-3b-4e1t,INT4-MIXED,1024,332.3,25,40.00 -qwen2.5-1.5b-instruct,INT8-CW,32,80.6,25.1,39.84 -phi-3-mini-128k-instruct,INT4-MIXED,32,50.7,25.2,39.68 -qwen2.5-1.5b-instruct,INT8-CW,1024,180.9,25.7,38.91 -tiny-llama-1.1b-chat,FP16,32,30.3,26.2,38.17 -phi-3-mini-4k-instruct,INT4-MIXED,1024,373.5,26.5,37.74 -phi-3-mini-128k-instruct,INT4-MIXED,1024,375.3,26.6,37.59 -phi-3.5-mini-instruct,INT4-MIXED,1024,372.5,26.7,37.45 -phi-3-mini-4k-instruct,INT4-MIXED,32,50.1,26.8,37.31 -qwen3-4b,INT4-MIXED,32,58.6,26.8,37.31 -tiny-llama-1.1b-chat,FP16,1024,113.8,26.9,37.17 -phi-3.5-mini-instruct,INT4-MIXED,32,53.5,27.1,36.90 -internvl2-4b,INT4-MIXED,297,221.1,27.3,36.63 -internvl2-4b,INT4-MIXED,297,215.9,27.5,36.36 -phi-4-mini-reasoning,INT4-MIXED,32,63.7,27.5,36.36 -phi-3.5-mini-instruct,INT4-MIXED,1024,391.6,27.6,36.23 -qwen3-4b,INT4-MIXED,32,60,27.6,36.23 -phi-4-mini-instruct,INT4-MIXED,32,64.5,27.7,36.10 -phi-3-mini-4k-instruct,INT4-MIXED,1024,391.1,28,35.71 -gemma-4-e2b-it,INT4-MIXED,274,426.9,28.1,35.59 -afm-4.5b,INT4-MIXED,32,51.9,28.2,35.46 -phi-3-mini-128k-instruct,INT4-MIXED,1024,488.9,28.2,35.46 -gpt-oss-20b,INT4-MIXED,32,207.2,28.3,35.34 -phi-4-mini-instruct,INT4-MIXED,32,64.5,28.4,35.21 -gemma-2b-it,INT8-CW,32,36.2,28.5,35.09 -phi-4-mini-reasoning,INT4-MIXED,32,65.6,28.5,35.09 -phi-4-mini-reasoning,INT4-MIXED,32,66.4,28.7,34.84 -gemma-3-4b-it,INT4-MIXED,32,82.9,28.9,34.60 -gemma-4-e2b-it,INT4-MIXED,274,457.9,28.9,34.60 -gemma-2b-it,INT8-CW,1024,211.4,29.4,34.01 -phi-3.5-vision-instruct,INT4-MIXED,802,489,29.5,33.90 -phi-4-mini-reasoning,INT4-MIXED,1024,375.3,29.5,33.90 -qwen3-4b,INT4-MIXED,1024,347,29.5,33.90 -afm-4.5b,INT4-MIXED,1024,391.1,29.7,33.67 -phi-3-mini-4k-instruct,INT4-MIXED,1024,524.3,29.7,33.67 -gemma-3-4b-it,INT4-MIXED,32,85.3,29.8,33.56 -phi-3.5-mini-instruct,INT4-MIXED,1024,548.9,29.8,33.56 -phi-4-mini-instruct,INT4-MIXED,1024,375.6,29.8,33.56 -llama-3.2-1b-instruct,FP16,32,35,29.9,33.44 -whisper-large-v3,INT4-MIXED,prompt0,285.2,29.9,33.44 -gemma-3-4b-it,INT4-MIXED,32,86,30.1,33.22 -qwen3-4b,INT4-MIXED,1024,384,30.1,33.22 -internvl2-4b,INT4-MIXED,1027,588.9,30.2,33.11 -glm-edge-4b-chat,INT4-MIXED,32,95,30.3,33.00 -phi-4-mini-reasoning,INT4-MIXED,1024,387.9,30.3,33.00 -phi-3.5-vision-instruct,INT4-MIXED,1032,607.6,30.4,32.89 -phi-4-mini-instruct,INT4-MIXED,1024,391.1,30.4,32.89 -gemma-3-4b-it,INT4-MIXED,1024,370.1,30.5,32.79 -internvl2-4b,INT4-MIXED,1027,589.7,30.5,32.79 -phi-4-mini-reasoning,INT4-MIXED,1024,407.8,30.5,32.79 -llama-3.2-1b-instruct,FP16,1024,135.5,30.6,32.68 -gpt-oss-20b,INT4-MIXED,1024,671.5,30.7,32.57 -whisper-large-v3,INT4-MIXED,prompt1,333.6,30.7,32.57 -gemma-2-2b,INT8-CW,33,47.4,30.8,32.47 -phi-2,INT8-CW,32,66.9,31.5,31.75 -whisper-large-v3,INT8-CW,prompt0,292.8,31.5,31.75 -stable-zephyr-3b-dpo,INT8-CW,32,65.3,31.6,31.65 -stablelm-3b-4e1t,INT8-CW,32,66.5,31.6,31.65 -gemma-3-4b-it,INT4-MIXED,1024,404.9,31.8,31.45 -whisper-large-v3,INT8-CW,prompt1,343.8,31.9,31.35 -gemma-3-4b-it,INT4-MIXED,1024,406.7,32.1,31.15 -glm-edge-4b-chat,INT4-MIXED,1024,571.8,32.4,30.86 -minicpm-1b-sft,FP16,31,60.5,32.5,30.77 -gemma-2-2b,INT8-CW,1025,240.2,32.6,30.67 -chatglm3-6b,INT4-MIXED,32,45.1,33.1,30.21 -gemma-4-e2b-it,INT4-MIXED,1024,601.1,33.4,29.94 -deepseek-r1-distill-qwen-1.5b,FP16,32,40.1,33.5,29.85 -qwen2.5-coder-1.5b-instruct,FP16,32,40.6,33.5,29.85 -qwen2.5-1.5b-instruct,FP16,32,40.1,33.6,29.76 -stablelm-3b-4e1t,INT8-CW,1024,340.3,33.8,29.59 -gemma-4-e2b-it,INT4-MIXED,1024,582.4,33.9,29.50 -qwen2.5-coder-3b-instruct,INT8-CW,32,58.2,33.9,29.50 -minicpm-1b-sft,FP16,1014,184.3,34,29.41 -stable-zephyr-3b-dpo,INT8-CW,1024,334.3,34,29.41 -phi-2,INT8-CW,1024,343.4,34.2,29.24 -qwen2.5-1.5b-instruct,FP16,1024,191.3,34.5,28.99 -qwen2.5-coder-1.5b-instruct,FP16,1024,165.6,34.6,28.90 -deepseek-r1-distill-qwen-1.5b,FP16,1024,167.1,34.8,28.74 -gpt-oss-20b,INT4-MIXED,32,218.1,34.8,28.74 -chatglm3-6b,INT4-MIXED,1024,470.8,35.2,28.41 -qwen2.5-coder-3b-instruct,INT8-CW,1024,276.6,35.2,28.41 -chatglm3-6b,INT4-MIXED,32,49.7,35.4,28.25 -gemma-4-e2b-it,INT8-CW,274,461.1,35.6,28.09 -phi-4-multimodal-instruct,INT4-MIXED,578,462.4,35.6,28.09 -phi-4-multimodal-instruct,INT4-MIXED,786,542.7,35.6,28.09 -whisper-large-v3,FP16,prompt0,276.7,35.8,27.93 -whisper-large-v3,FP16,prompt1,315.9,36.2,27.62 -gpt-oss-20b,INT4-MIXED,1024,701.4,36.3,27.55 -phi-4-multimodal-instruct,INT4-MIXED,1362,1130.2,36.3,27.55 -phi-4-multimodal-instruct,INT4-MIXED,1570,1249.6,36.3,27.55 -llama-2-7b-chat-hf,INT4-MIXED,32,53.2,36.8,27.17 -codellama-7b,INT4-MIXED,32,52.4,36.9,27.10 -chatglm3-6b,INT4-MIXED,1024,558,37.2,26.88 -llama-3.2-3b-instruct,INT8-CW,32,46.8,37.3,26.81 -llama-3.2-3b-instruct,INT8-CW,1024,284.3,38.5,25.97 -biomistral-7b-slerp,INT4-MIXED,7,45.4,38.7,25.84 -mistral-7b-instruct-v0.2,INT4-MIXED,32,54,38.8,25.77 -zephyr-7b-beta,INT4-MIXED,32,54.1,38.8,25.77 -llama-2-7b-chat-hf,INT4-MIXED,32,55.6,38.9,25.71 -neural-chat-7b-v3-3,INT4-MIXED,32,53.7,39.1,25.58 -mistral-7b-instruct-v0.3,INT4-MIXED,32,54.2,39.2,25.51 -glm-edge-1.5b-chat,FP16,32,52.9,39.3,25.45 -codellama-7b,INT4-MIXED,32,55.9,39.4,25.38 -qwen3_8b_eagle3,INT4-MIXED,32,65.7,39.7,25.19 -falcon-7b-instruct,INT4-MIXED,32,56.1,40.2,24.88 -glm-edge-1.5b-chat,FP16,1024,218.3,40.4,24.75 -llama-2-7b-chat-hf,INT4-MIXED,1024,530.3,40.6,24.63 -codellama-7b,INT4-MIXED,1024,523.7,40.7,24.57 -minicpm3-4b,INT4-MIXED,32,178.7,40.7,24.57 -phi-4-multimodal-instruct,INT4-MIXED,578,524.2,40.8,24.51 -mistral-7b-instruct-v0.2,INT4-MIXED,32,59,40.9,24.45 -mistral-7b-instruct-v0.3,INT4-MIXED,32,58.7,40.9,24.45 -qwen3-vl-4b-instruct,INT4-MIXED,4937,6482,40.9,24.45 -phi-4-multimodal-instruct,INT4-MIXED,786,645.3,41,24.39 -qwen3-vl-4b-instruct,INT4-MIXED,4907,6208.2,41,24.39 -stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,42.5,41,24.39 -minicpm3-4b,INT4-MIXED,32,174.9,41.1,24.33 -biomistral-7b-slerp,INT4-MIXED,7,46.8,41.3,24.21 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,545,41.3,24.21 -neural-chat-7b-v3-3,INT4-MIXED,1024,540.6,41.3,24.21 -zephyr-7b-beta,INT4-MIXED,1024,535.1,41.3,24.21 -mistral-7b-instruct-v0.1,INT4-MIXED,32,59.2,41.4,24.15 -mistral-7b-instruct-v0.2,INT4-MIXED,32,59,41.4,24.15 -mistral-7b-instruct-v0.3,INT4-MIXED,32,58.1,41.4,24.15 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,540.4,41.5,24.10 -neural-chat-7b-v3-3,INT4-MIXED,32,58.5,41.5,24.10 -minicpm3-4b,INT4-MIXED,32,174.7,41.6,24.04 -qwen3-vl-4b-instruct,INT4-MIXED,4907,6404.3,41.8,23.92 -qwen3-vl-4b-instruct,INT4-MIXED,4907,6597.4,42,23.81 -falcon-7b-instruct,INT4-MIXED,1024,570.2,42.1,23.75 -qwen2.5-7b-instruct-1m,INT4-MIXED,32,69.4,42.1,23.75 -qwen3-vl-4b-instruct,INT4-MIXED,4937,6814.4,42.3,23.64 -qwen3-vl-4b-instruct,INT4-MIXED,4937,6576.2,42.3,23.64 -phi-4-multimodal-instruct,INT4-MIXED,1570,1439.4,42.4,23.58 -llama-2-7b-chat-hf,INT4-MIXED,1024,585.5,42.5,23.53 -qwen2.5-7b-instruct,INT4-MIXED,32,56,42.5,23.53 -qwen2.5-7b-instruct,INT4-MIXED,32,56.8,42.5,23.53 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,57.2,42.6,23.47 -phi-4-multimodal-instruct,INT4-MIXED,1362,1299.1,42.6,23.47 -llama-3-8b-instruct,INT4-MIXED,32,60.4,42.9,23.31 -gemma-4-e2b-it,INT8-CW,1024,651.1,43,23.26 -llama-3.1-8b-instruct,INT4-MIXED,32,58.3,43,23.26 -codellama-7b,INT4-MIXED,1024,610.4,43.1,23.20 -phi-3-mini-128k-instruct,INT8-CW,32,56.9,43.1,23.20 -phi-3-mini-4k-instruct,INT8-CW,32,56.5,43.1,23.20 -phi-3.5-mini-instruct,INT8-CW,32,56.8,43.1,23.20 -qwen3_8b_eagle3,INT4-MIXED,1024,637.5,43.2,23.15 -stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,44.4,43.2,23.15 -minicpm-v-2_6,INT4-MIXED,228,544.7,43.3,23.09 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,576.6,43.3,23.09 -minicpm3-4b,INT4-MIXED,1024,723.6,43.4,23.04 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,576.7,43.4,23.04 -qwen2.5-7b-instruct-1m,INT4-MIXED,1024,489.2,43.4,23.04 -qwen2.5-7b-instruct,INT4-MIXED,32,59.8,43.5,22.99 -qwen2.5-7b-instruct,INT4-MIXED,32,60,43.5,22.99 -deepseek-r1-distill-qwen-7b,INT4-MIXED,32,60.3,43.6,22.94 -minicpm-o-2_6,INT4-MIXED,238,559.6,43.6,22.94 -qwen2-7b-instruct,INT4-MIXED,32,59.5,43.6,22.94 -qwen2.5-7b-instruct,INT4-MIXED,1024,486.4,43.6,22.94 -qwen2.5-7b-instruct,INT4-MIXED,1024,489.8,43.7,22.88 -qwen2.5-7b-instruct,INT4-MIXED,32,59.9,43.8,22.83 -mistral-7b-instruct-v0.1,INT4-MIXED,1025,695.9,43.9,22.78 -neural-chat-7b-v3-3,INT4-MIXED,1024,648,43.9,22.78 -deepseek-r1-distill-qwen-7b,INT4-MIXED,32,60.4,44,22.73 -minicpm3-4b,INT4-MIXED,1024,860.6,44,22.73 -mistral-7b-instruct-v0.2,INT4-MIXED,1024,647.7,44,22.73 -mistral-7b-instruct-v0.3,INT4-MIXED,1024,644.5,44,22.73 -qwen2-7b-instruct,INT4-MIXED,32,60.2,44.1,22.68 -qwen3-8b,INT4-MIXED,32,71,44.1,22.68 -qwen2.5-7b-instruct,INT4-MIXED,32,60,44.2,22.62 -qwen2.5-7b-instruct-1m,INT4-MIXED,32,59.7,44.3,22.57 -bloomz-7b1,INT4-MIXED,32,60.8,44.4,22.52 -minicpm3-4b,INT4-MIXED,1024,756.3,44.5,22.47 -minicpm4-8b,INT4-MIXED,32,63.8,44.5,22.47 -falcon-7b-instruct,INT4-MIXED,32,62.5,44.7,22.37 -llama-3-8b-instruct,INT4-MIXED,32,62.6,44.7,22.37 -minicpm-v-2_6,INT4-MIXED,228,554.2,44.7,22.37 -phi-4-mini-reasoning,INT8-CW,32,68.5,44.7,22.37 -llama-3.1-8b-instruct,INT4-MIXED,32,62.4,44.8,22.32 -gemma-3-4b-it,INT8-CW,32,89.8,44.9,22.27 -llama-3-8b-instruct,INT4-MIXED,32,62.9,45,22.22 -phi-4-mini-instruct,INT8-CW,32,69.1,45,22.22 -qwen2.5-7b-instruct,INT4-MIXED,1024,550.2,45,22.22 -qwen2.5-7b-instruct,INT4-MIXED,1024,544.7,45,22.22 -deepseek-r1-distill-llama-8b,INT4-MIXED,1024,547,45.1,22.17 -deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,527.9,45.1,22.17 -minicpm-v-2_6,INT4-MIXED,228,579.9,45.1,22.17 -qwen2-7b-instruct,INT4-MIXED,1024,522.2,45.1,22.17 -minicpm-o-2_6,INT4-MIXED,238,570,45.3,22.08 -deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,583.6,45.4,22.03 -internvl2-4b,INT8-CW,297,215.6,45.4,22.03 -llama-3-8b-instruct,INT4-MIXED,32,62.6,45.4,22.03 -qwen2.5-7b-instruct,INT4-MIXED,1024,580.4,45.4,22.03 -qwen2-7b-instruct,INT4-MIXED,1024,587.9,45.5,21.98 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,61.3,45.6,21.93 -llama-3.1-8b-instruct,INT4-MIXED,1024,544.8,45.8,21.83 -phi-3-mini-128k-instruct,INT8-CW,1024,490.4,45.8,21.83 -qwen2.5-7b-instruct-1m,INT4-MIXED,1024,587.3,45.8,21.83 -phi-3-mini-4k-instruct,INT8-CW,1024,484.2,45.9,21.79 -phi-3.5-mini-instruct,INT8-CW,1024,487.7,45.9,21.79 -qwen2.5-7b-instruct,INT4-MIXED,1024,589,46,21.74 -qwen3-4b,INT8-CW,32,64.1,46,21.74 -minicpm-v-4_5,INT4-MIXED,217,593.1,46.1,21.69 -phi-4-mini-instruct,INT8-CW,1024,420.8,46.4,21.55 -phi-4-mini-reasoning,INT8-CW,1024,421,46.4,21.55 -minicpm4-8b,INT4-MIXED,1024,577.6,46.5,21.51 -qwen3-8b,INT4-MIXED,32,70.6,46.5,21.51 -bloomz-7b1,INT4-MIXED,32,65.9,46.7,21.41 -minicpm4-8b,INT4-MIXED,32,70.4,46.8,21.37 -qwen3-8b,INT4-MIXED,1024,565.3,46.8,21.37 -falcon-7b-instruct,INT4-MIXED,1024,666.3,46.9,21.32 -qwen3-8b,INT4-MIXED,32,73.4,46.9,21.32 -gemma-3-4b-it,INT8-CW,1024,418.2,47,21.28 -llama-3-8b-instruct,INT4-MIXED,1024,578.7,47.2,21.19 -llama-3-8b-instruct,INT4-MIXED,1024,582.2,47.3,21.14 -llama-3.1-8b-instruct,INT4-MIXED,1024,577,47.3,21.14 -phi-3.5-vision-instruct,INT8-CW,802,501.9,47.3,21.14 -llama-3-8b-instruct,INT4-MIXED,1024,549.1,47.6,21.01 -afm-4.5b,INT8-CW,32,60.1,47.7,20.96 -bloomz-7b1,INT4-MIXED,1024,686.7,47.7,20.96 -deepseek-r1-distill-llama-8b,INT4-MIXED,1024,568.9,48,20.83 -llama-3-8b-instruct,INT4-MIXED,1024,648.1,48.2,20.75 -minicpm4-8b,INT4-MIXED,32,69.2,48.3,20.70 -internvl2-4b,INT8-CW,1027,606.8,48.4,20.66 -afm-4.5b,INT8-CW,1024,385.7,48.5,20.62 -minicpm-v-4_5,INT4-MIXED,217,614.8,48.5,20.62 -qwen3-4b,INT8-CW,1024,407.4,48.6,20.58 -phi-3.5-vision-instruct,INT8-CW,1032,633.2,48.7,20.53 -minicpm4-8b,INT4-MIXED,1024,620.6,48.9,20.45 -qwen3-8b,INT4-MIXED,1024,600.7,49.1,20.37 -qwen3-8b,INT4-MIXED,1024,675.1,49.8,20.08 -glm-4-9b-chat-hf,INT4-MIXED,32,67.1,49.9,20.04 -zephyr-7b-beta,INT4-MIXED,32,64.6,49.9,20.04 -qwen2.5-vl-7b-instruct,INT4-MIXED,32,150.7,50.1,19.96 -bloomz-7b1,INT4-MIXED,1024,797,50.2,19.92 -glm-edge-4b-chat,INT8-CW,32,96,50.3,19.88 -minicpm4-8b,INT4-MIXED,1024,713.5,50.5,19.80 -gemma-7b-it,INT4-MIXED,32,71.7,51,19.61 -baichuan2-7b-chat,INT4-MIXED,32,66.5,51.1,19.57 -qwen2.5-vl-7b-instruct,INT4-MIXED,1024,615.4,51.4,19.46 -glm-4-9b-chat-hf,INT4-MIXED,32,73.4,51.9,19.27 -glm-edge-4b-chat,INT8-CW,1024,586.2,52.4,19.08 -zephyr-7b-beta,INT4-MIXED,1024,570.5,52.4,19.08 -glm-4-9b-chat-hf,INT4-MIXED,1024,671.6,52.5,19.05 -glm-4-9b-chat-hf,INT4-MIXED,32,73,53.1,18.83 -fara-7b,INT4-MIXED,32,174.9,53.5,18.69 -gemma-7b-it,INT4-MIXED,32,76.1,53.9,18.55 -qwen2.5-vl-7b-instruct,INT4-MIXED,32,150.8,54,18.52 -qwen2.5-vl-7b-instruct,INT4-MIXED,1024,610.3,54.1,18.48 -phi-4-multimodal-instruct,INT8-CW,578,497.1,54.4,18.38 -qwen2.5-vl-7b-instruct,INT4-MIXED,1024,674.8,54.6,18.32 -qwen2.5-vl-7b-instruct,INT4-MIXED,32,153.9,54.8,18.25 -phi-4-multimodal-instruct,INT8-CW,786,602.7,54.9,18.21 -baichuan2-7b-chat,INT4-MIXED,1024,646.9,55,18.18 -glm-4-9b-chat-hf,INT4-MIXED,1024,703.5,55,18.18 -gemma-7b-it,INT4-MIXED,1024,645,55.1,18.15 -qwen3.5-9b,INT4-MIXED,83,251.7,55.1,18.15 -ltx-video,INT4-MIXED,11,55.7,55.2,18.12 -llava-next-video-7b-hf,INT4-MIXED,2945,3184.7,56.1,17.83 -minicpm3-4b,INT8-CW,32,200.7,56.1,17.83 -fara-7b,INT4-MIXED,1024,839,56.2,17.79 -glm-4-9b-chat-hf,INT4-MIXED,1024,785.2,56.2,17.79 -phi-4-multimodal-instruct,INT8-CW,1362,1239.2,56.2,17.79 -qwen3.5-9b,INT4-MIXED,1024,1000.9,56.3,17.76 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,73.4,56.4,17.73 -phi-4-multimodal-instruct,INT8-CW,1570,1378,56.5,17.70 -llama-3.1-8b-instruct,INT4-MIXED,32,73.2,56.6,17.67 -phi-2,FP16,32,65.1,57.2,17.48 -gemma-7b-it,INT4-MIXED,1024,753,57.8,17.30 -ltx-video,INT8-CW,11,57.6,57.8,17.30 -gemma-4-e2b-it,FP16,274,504.3,57.9,17.27 -stable-zephyr-3b-dpo,FP16,32,65.1,58,17.24 -gemma-2-9b-it,INT4-MIXED,32,80.4,58.2,17.18 -deepseek-r1-distill-llama-8b,INT4-MIXED,1024,718.3,58.7,17.04 -llama-3.1-8b-instruct,INT4-MIXED,1024,719.3,58.8,17.01 -qwen3.5-9b,INT4-MIXED,83,259.5,58.8,17.01 -gemma-2-2b,FP16,33,64,58.9,16.98 -stablelm-3b-4e1t,FP16,32,65,58.9,16.98 -qwen3.5-9b,INT4-MIXED,1024,1162.2,59,16.95 -gemma-2-9b-it,INT4-MIXED,32,82.7,59.8,16.72 -qwen3-vl-4b-instruct,INT8-CW,4907,6586.4,59.8,16.72 -gemma-2-2b,FP16,1025,334.1,60.3,16.58 -qwen3-vl-4b-instruct,INT8-CW,4937,6840.8,60.4,16.56 -phi-2,FP16,1024,433.8,61.1,16.37 -gemma-2-9b-it,INT4-MIXED,32,80.7,61.2,16.34 -stable-zephyr-3b-dpo,FP16,1024,411.6,61.7,16.21 -minicpm3-4b,INT8-CW,1024,910.9,61.8,16.18 -stablelm-3b-4e1t,FP16,1024,411.9,61.8,16.18 -gemma-2b-it,FP16,32,67.2,61.9,16.16 -gemma-2-9b-it,INT4-MIXED,1024,724.6,62.2,16.08 -gemma-2b-it,FP16,1024,256.5,62.5,16.00 -gemma-2-9b-it,INT4-MIXED,1024,750.2,64.3,15.55 -gemma-2-9b-it,INT4-MIXED,1024,814.2,65.5,15.27 -gemma-4-e2b-it,FP16,1024,739,65.6,15.24 -lcm-dreamshaper-v7,INT8-HYBRID,1024,70.5,66.2,15.11 -lcm-dreamshaper-v7,INT8-HYBRID,32,69.7,66.6,15.02 -chatglm3-6b,INT8-CW,32,80.5,66.9,14.95 -llama-3.2-3b-instruct,FP16,32,72.5,67.6,14.79 -qwen3_8b_eagle3,INT8-CW,1024,673.7,67.6,14.79 -dolly-v2-12b,INT4-MIXED,32,102.1,68,14.71 -chatglm3-6b,INT8-CW,1024,548.8,68.7,14.56 -llama-3.2-3b-instruct,FP16,1024,403.5,69.4,14.41 -llama-2-13b-chat-hf,INT4-MIXED,32,96.6,69.9,14.31 -qwen2.5-coder-3b-instruct,FP16,32,78,72.8,13.74 -dolly-v2-12b,INT4-MIXED,1024,1248.3,73,13.70 -gemma-3-12b-it,INT4-MIXED,32,138.4,73,13.70 -qwen3_8b_eagle3,INT8-CW,32,105.5,73.1,13.68 -falcon-7b-instruct,INT8-CW,32,89.3,74.1,13.50 -qwen2.5-coder-3b-instruct,FP16,1024,362.8,74.5,13.42 -qwen2.5-7b-instruct,INT8-CW,32,94.4,74.7,13.39 -qwen2.5-7b-instruct-1m,INT8-CW,32,94.4,74.7,13.39 -qwen2.5-7b-instruct,INT8-CW,32,95.2,74.9,13.35 -qwen2-7b-instruct,INT8-CW,32,94.8,75,13.33 -llama-2-13b-chat-hf,INT4-MIXED,1024,1015.5,75.5,13.25 -deepseek-r1-distill-qwen-7b,INT8-CW,32,94.9,75.8,13.19 -minicpm-v-2_6,INT8-CW,228,551.9,75.9,13.18 -falcon-7b-instruct,INT8-CW,1024,814,76,13.16 -codellama-7b,INT8-CW,32,88.2,76.2,13.12 -qwen2.5-7b-instruct,INT8-CW,1024,581.9,76.2,13.12 -qwen2-7b-instruct,INT8-CW,1024,580.4,76.3,13.11 -llama-2-7b-chat-hf,INT8-CW,32,87.9,76.4,13.09 -qwen2.5-7b-instruct,INT8-CW,1024,581.1,76.4,13.09 -gemma-3-12b-it,INT4-MIXED,32,145.6,76.7,13.04 -phi-3.5-mini-instruct,FP16,32,84.5,76.7,13.04 -phi-3-mini-128k-instruct,FP16,32,84.7,76.8,13.02 -gemma-3-12b-it,INT4-MIXED,1024,1042.3,76.9,13.00 -phi-3-mini-4k-instruct,FP16,32,84.8,76.9,13.00 -deepseek-r1-distill-qwen-7b,INT8-CW,1024,585.3,77.1,12.97 -qwen2.5-7b-instruct-1m,INT8-CW,1024,576.2,77.2,12.95 -minicpm-o-2_6,INT8-CW,238,564.2,77.4,12.92 -phi-4,INT4-MIXED,32,111.2,77.4,12.92 -lcm-dreamshaper-v7,INT8-CW,1024,80.1,77.9,12.84 -lcm-dreamshaper-v7,INT8-CW,32,79,78.1,12.80 -phi-4-reasoning,INT4-MIXED,32,114.1,78.9,12.67 -internvl2-4b,FP16,297,283.7,79.1,12.64 -codellama-7b,INT8-CW,1024,580.4,79.6,12.56 -qwen1.5-14b-chat,INT4-MIXED,32,114.9,79.6,12.56 -llama-2-7b-chat-hf,INT8-CW,1024,593.3,79.7,12.55 -lcm-dreamshaper-v7,INT8-CW,1024,80.4,79.8,12.53 -lcm-dreamshaper-v7,INT8-CW,32,80.7,79.8,12.53 -phi-4-mini-instruct,FP16,32,89.1,79.8,12.53 -phi-4-mini-reasoning,FP16,32,89.6,79.9,12.52 -gemma-3-12b-it,INT4-MIXED,1024,1162.9,80.8,12.38 -phi-3.5-vision-instruct,FP16,802,678.1,80.8,12.38 -phi-4-reasoning,INT4-MIXED,32,122.9,80.9,12.36 -phi-3.5-mini-instruct,FP16,1024,575.1,81,12.35 -mistral-7b-instruct-v0.1,INT8-CW,32,95.3,81.1,12.33 -phi-3-mini-128k-instruct,FP16,1024,576.8,81.1,12.33 -phi-4,INT4-MIXED,1024,1142.3,81.1,12.33 -zephyr-7b-beta,INT8-CW,32,94.4,81.1,12.33 -neural-chat-7b-v3-3,INT8-CW,32,94.2,81.2,12.32 -mistral-7b-instruct-v0.3,INT8-CW,32,94.1,81.3,12.30 -mistral-7b-instruct-v0.2,INT8-CW,32,93.6,81.4,12.29 -phi-3-mini-4k-instruct,FP16,1024,572.6,81.4,12.29 -internvl2-4b,FP16,1027,850,81.6,12.25 -phi-4,INT4-MIXED,32,123.8,81.6,12.25 -phi-3.5-vision-instruct,FP16,1032,866.4,81.8,12.22 -biomistral-7b-slerp,INT8-CW,7,85.7,81.9,12.21 -phi-4-mini-instruct,FP16,1024,552.3,82,12.20 -phi-4-mini-reasoning,FP16,1024,557,82.2,12.17 -phi-4-reasoning,INT4-MIXED,1024,1161.8,82.7,12.09 -baichuan2-7b-chat,INT8-CW,32,93.4,82.8,12.08 -gemma-3-4b-it,FP16,32,105.2,82.8,12.08 -deepseek-r1-distill-qwen-14b,INT4-MIXED,32,113.2,83.1,12.03 -mistral-7b-instruct-v0.3,INT8-CW,1024,609.4,83.4,11.99 -fara-7b,INT8-CW,32,215.4,83.5,11.98 -mistral-7b-instruct-v0.1,INT8-CW,1025,656.8,83.5,11.98 -zephyr-7b-beta,INT8-CW,1024,605.3,83.5,11.98 -qwen2.5-vl-7b-instruct,INT8-CW,32,191.2,83.7,11.95 -mistral-7b-instruct-v0.2,INT8-CW,1024,606.4,83.8,11.93 -neural-chat-7b-v3-3,INT8-CW,1024,603.8,83.8,11.93 -bloomz-7b1,INT8-CW,32,96.2,84.3,11.86 -gemma-3-4b-it,FP16,1024,550.8,84.3,11.86 -fara-7b,INT8-CW,1024,835,84.9,11.78 -llama-3.1-8b-instruct,INT8-CW,32,98.2,85,11.76 -llama-3-8b-instruct,INT8-CW,32,98.5,85.2,11.74 -phi-4,INT4-MIXED,1024,1326.6,85.3,11.72 -qwen1.5-14b-chat,INT4-MIXED,1024,1166,85.4,11.71 -baichuan2-7b-chat,INT8-CW,1024,645.6,85.5,11.70 -phi-4-reasoning,INT4-MIXED,1024,1206.1,85.6,11.68 -qwen3-4b,FP16,32,93.8,85.7,11.67 -phi-4-reasoning,INT4-MIXED,32,123.5,86.2,11.60 -deepseek-r1-distill-llama-8b,INT8-CW,32,98.5,86.3,11.59 -qwen2.5-vl-7b-instruct,INT8-CW,1024,697.1,86.7,11.53 -bloomz-7b1,INT8-CW,1024,798.3,86.9,11.51 -deepseek-r1-distill-qwen-14b,INT4-MIXED,1024,1217.4,87.2,11.47 -glm-edge-4b-chat,FP16,32,100.5,87.2,11.47 -llama-3-8b-instruct,INT8-CW,1024,619.3,87.6,11.42 -llama-3.1-8b-instruct,INT8-CW,1024,614.4,87.6,11.42 -qwen3-4b,FP16,1024,553.4,88.5,11.30 -deepseek-r1-distill-llama-8b,INT8-CW,1024,616.3,88.8,11.26 -llava-v1.6-mistral-7b-hf,INT8-CW,2944,2916.3,88.8,11.26 -starcoder,INT4-MIXED,32,140.3,89.2,11.21 -glm-edge-4b-chat,FP16,1024,704.6,89.7,11.15 -llama-2-13b-chat-hf,INT4-MIXED,32,120.4,89.7,11.15 -phi-4-reasoning,INT4-MIXED,1024,1371.4,89.9,11.12 -starcoder2-15b,INT4-MIXED,32,144.6,90.7,11.03 -afm-4.5b,FP16,32,100.3,90.9,11.00 -qwen3-8b,INT8-CW,32,101.4,91.3,10.95 -afm-4.5b,FP16,1024,498,91.8,10.89 -minicpm-v-4_5,INT8-CW,217,586.8,91.9,10.88 -starcoder,INT4-MIXED,1024,1437.6,92.8,10.78 -qwen3-8b,INT8-CW,1024,657.4,92.9,10.76 -llava-next-video-7b-hf,INT8-CW,2945,3258.6,93.1,10.74 -gemma-7b-it,INT8-CW,32,114.5,93.2,10.73 -llama-2-13b-chat-hf,INT4-MIXED,1024,1204,93.9,10.65 -minicpm4-8b,INT8-CW,32,106.7,94.6,10.57 -starcoder2-15b,INT4-MIXED,1024,1455.4,94.6,10.57 -phi-4-multimodal-instruct,FP16,578,629.7,94.7,10.56 -phi-4-multimodal-instruct,FP16,786,767.6,95,10.53 -minicpm3-4b,FP16,32,177.5,95.7,10.45 -minicpm4-8b,INT8-CW,1024,716.2,96.5,10.36 -glm-4-9b-chat-hf,INT8-CW,32,114.3,97.3,10.28 -lcm-dreamshaper-v7,FP16,32,86.1,97.3,10.28 -gemma-7b-it,INT8-CW,1024,757.2,97.7,10.24 -lcm-dreamshaper-v7,FP16,1024,94.1,97.7,10.24 -phi-4-multimodal-instruct,FP16,1570,1675,97.9,10.21 -phi-4-multimodal-instruct,FP16,1362,1524.4,98.6,10.14 \ No newline at end of file +lfm2.5-350m,INT8-CW,32,20.2,5,200 +lfm2.5-350m,INT8-CW,1024,40,5.1,196.0784314 +lfm2.5-350m,INT4-MIXED,1024,40.8,5.2,192.3076923 +lfm2.5-350m,INT4-MIXED,32,19,5.2,192.3076923 +distil-large-v2,INT4-MIXED,32,165.8,5.9,169.4915254 +distil-large-v2,INT4-MIXED,1024,213.2,6,166.6666667 +distil-large-v2,INT8-CW,32,164.1,6,166.6666667 +distil-large-v2,INT8-CW,1024,209.4,6,166.6666667 +whisper-large-v3-turbo,INT8-CW,1024,225.7,6.3,158.7301587 +whisper-large-v3-turbo,INT8-CW,32,176.3,6.4,156.25 +whisper-large-v3-turbo,INT4-MIXED,32,183.8,7,142.8571429 +whisper-large-v3-turbo,INT4-MIXED,1024,235.6,7,142.8571429 +minicpm4-0.5b,INT4-MIXED,32,28.3,7.4,135.1351351 +minicpm4-0.5b,INT4-MIXED,1024,43.9,7.4,135.1351351 +minicpm4-0.5b,INT4-MIXED,32,27.9,7.4,135.1351351 +gemma-3-270m,INT4-MIXED,32,29,7.5,133.3333333 +minicpm4-0.5b,INT4-MIXED,1024,41,7.5,133.3333333 +tiny-llama-1.1b-chat,INT4-MIXED,32,25,7.5,133.3333333 +gemma-3-270m,INT8-CW,1024,35.7,7.6,131.5789474 +minicpm4-0.5b,INT8-CW,32,31.4,7.7,129.8701299 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,28.6,7.7,129.8701299 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,45.4,7.7,129.8701299 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,28.6,7.7,129.8701299 +minicpm4-0.5b,INT4-MIXED,32,30.2,7.8,128.2051282 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,42.8,7.8,128.2051282 +minicpm4-0.5b,INT4-MIXED,1024,50.1,7.9,126.5822785 +lfm2.5-350m,FP16,32,16.7,8,125 +minicpm4-0.5b,INT8-CW,1024,49.7,8,125 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,30,8,125 +lfm2.5-1.2b-instruct,INT4-MIXED,32,20,8.1,123.4567901 +lfm2.5-350m,FP16,1024,41,8.1,123.4567901 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,52.5,8.1,123.4567901 +gemma-3-270m,INT8-CW,32,30.8,8.2,121.9512195 +nuextract-tiny,INT4-MIXED,32,30.5,8.2,121.9512195 +tiny-llama-1.1b-chat,INT4-MIXED,32,26.4,8.2,121.9512195 +tiny-llama-1.1b-chat,INT4-MIXED,1024,81.6,8.2,121.9512195 +gemma-3-270m,FP16,1024,29.1,8.3,120.4819277 +gemma-3-270m,FP16,32,25.1,8.3,120.4819277 +lfm2-1.2b,INT4-MIXED,32,20.1,8.3,120.4819277 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,81.6,8.3,120.4819277 +lfm2.5-1.2b-instruct,INT4-MIXED,32,20.4,8.3,120.4819277 +nuextract-tiny,INT4-MIXED,1024,50.9,8.3,120.4819277 +qwen2.5-coder-0.5b-instruct,INT8-CW,32,30.7,8.3,120.4819277 +whisper-large-v3-turbo,FP16,32,177.3,8.3,120.4819277 +whisper-large-v3-turbo,FP16,1024,224.5,8.3,120.4819277 +lfm2-1.2b,INT4-MIXED,32,20.3,8.4,119.047619 +qwen2.5-coder-0.5b-instruct,INT8-CW,1024,49.7,8.4,119.047619 +lfm2-1.2b,INT4-MIXED,32,21.2,8.5,117.6470588 +lfm2-1.2b,INT4-MIXED,1024,80.9,8.5,117.6470588 +lfm2.5-1.2b-instruct,INT4-MIXED,32,21.4,8.5,117.6470588 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,86,8.5,117.6470588 +distil-large-v2,FP16,32,167.8,8.6,116.2790698 +lfm2-1.2b,INT4-MIXED,1024,85.8,8.6,116.2790698 +distil-large-v2,FP16,1024,214.4,8.7,114.9425287 +lfm2-1.2b,INT4-MIXED,1024,96.9,8.7,114.9425287 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,95.8,8.7,114.9425287 +nuextract-tiny,INT8-CW,32,31.1,8.9,112.3595506 +tiny-llama-1.1b-chat,INT4-MIXED,1024,106,8.9,112.3595506 +gemma-3-270m,INT4-MIXED,1024,32.9,9.1,109.8901099 +nuextract-tiny,INT8-CW,1024,47.8,9.1,109.8901099 +minicpm5-1b,INT4-MIXED,32,29.7,9.5,105.2631579 +minicpm5-1b,INT4-MIXED,1024,78.1,9.6,104.1666667 +llama-3.2-1b-instruct,INT4-MIXED,32,21.7,9.7,103.0927835 +llama-3.2-1b-instruct,INT4-MIXED,32,21.3,9.7,103.0927835 +llama-3.2-1b-instruct,INT4-MIXED,1024,82.2,10.2,98.03921569 +llama-3.2-1b-instruct,INT4-MIXED,1024,91.9,10.3,97.08737864 +whisper-small,INT4-MIXED,1024,139.3,10.5,95.23809524 +whisper-small,INT4-MIXED,32,94.6,10.6,94.33962264 +whisper-small,INT4-MIXED,32,103.7,10.8,92.59259259 +whisper-small,INT4-MIXED,1024,145.3,10.8,92.59259259 +minicpm4-0.5b,FP16,32,24.9,10.9,91.74311927 +nuextract-tiny,FP16,32,26.9,11.1,90.09009009 +whisper-small,INT8-CW,32,101.5,11.1,90.09009009 +lfm2-8b-a1b,INT4-MIXED,32,119,11.2,89.28571429 +minicpm4-0.5b,FP16,1024,55.8,11.3,88.49557522 +minicpm5-1b,INT8-CW,32,31,11.4,87.71929825 +whisper-small,INT8-CW,1024,144.8,11.4,87.71929825 +gemma-3-1b-it,INT4-MIXED,1024,83.1,11.6,86.20689655 +gemma-3-1b-it,INT4-MIXED,1024,85.7,11.7,85.47008547 +gemma-3-1b-it,INT4-MIXED,32,38.1,11.8,84.74576271 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,33.9,11.8,84.74576271 +gemma-3-1b-it,INT4-MIXED,32,37.6,11.9,84.03361345 +minicpm5-1b,INT8-CW,1024,77.5,11.9,84.03361345 +qwen2.5-1.5b-instruct,INT4-MIXED,32,34.2,11.9,84.03361345 +tiny-llama-1.1b-chat,INT8-CW,32,28.1,11.9,84.03361345 +exaone-4.0-1.2b,INT4-MIXED,32,42.1,12,83.33333333 +exaone-4.0-1.2b,INT4-MIXED,32,41.8,12,83.33333333 +gemma-3-1b-it,INT4-MIXED,32,39.5,12,83.33333333 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,35.4,12,83.33333333 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,34.2,12,83.33333333 +whisper-small,FP16,32,93,12,83.33333333 +nuextract-tiny,FP16,1024,54.7,12.1,82.6446281 +exaone-4.0-1.2b,INT4-MIXED,1024,131.2,12.2,81.96721311 +exaone-4.0-1.2b,INT4-MIXED,1024,133.4,12.2,81.96721311 +lfm2-8b-a1b,INT4-MIXED,1024,253.6,12.2,81.96721311 +nanollava,INT4-MIXED,760,97,12.2,81.96721311 +qwen2.5-coder-0.5b-instruct,FP16,32,27.2,12.2,81.96721311 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,108.7,12.2,81.96721311 +whisper-small,FP16,1024,138.4,12.2,81.96721311 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,39.2,12.3,81.30081301 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,132.3,12.4,80.64516129 +qwen2.5-1.5b-instruct,INT4-MIXED,32,34.3,12.4,80.64516129 +qwen2.5-coder-0.5b-instruct,FP16,1024,51,12.4,80.64516129 +exaone-4.0-1.2b,INT4-MIXED,32,44.6,12.5,80 +gemma-3-1b-it,INT4-MIXED,1024,91.5,12.5,80 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,139,12.5,80 +tiny-llama-1.1b-chat,INT8-CW,1024,93.8,12.5,80 +exaone-4.0-1.2b,INT4-MIXED,1024,142.1,12.6,79.36507937 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,133.1,12.6,79.36507937 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,131.2,12.6,79.36507937 +whisper-small,INT4-MIXED,32,101.9,12.8,78.125 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,132.2,13,76.92307692 +qwen2.5-1.5b-instruct,INT4-MIXED,32,36.9,13,76.92307692 +qwen2.5-1.5b-instruct,INT4-MIXED,32,36.9,13,76.92307692 +nanollava,FP16,760,105.6,13.1,76.33587786 +whisper-small,INT4-MIXED,1024,147.9,13.1,76.33587786 +nanollava,INT4-MIXED,1752,191.4,13.3,75.18796992 +nanollava,INT8-CW,760,100.1,13.5,74.07407407 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,141.7,13.5,74.07407407 +lfm2.5-1.2b-instruct,INT8-CW,32,22.6,13.6,73.52941176 +nanollava,INT8-CW,1752,183.8,13.6,73.52941176 +gemma-3-1b-it,INT8-CW,32,40.6,13.7,72.99270073 +lfm2.5-1.2b-instruct,INT8-CW,1024,106.2,13.8,72.46376812 +lfm2-1.2b,INT8-CW,32,22.8,13.9,71.94244604 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,140.3,13.9,71.94244604 +lfm2-1.2b,INT8-CW,1024,107,14.1,70.92198582 +gemma-3-1b-it,INT8-CW,1024,92.7,14.3,69.93006993 +llama-3.2-1b-instruct,INT8-CW,32,24.5,14.5,68.96551724 +nanollava,FP16,1752,200.2,14.6,68.49315068 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,37.4,15.5,64.51612903 +llama-3.2-1b-instruct,INT8-CW,1024,97.3,15.5,64.51612903 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,146.5,15.9,62.89308176 +minicpm-1b-sft,INT4-MIXED,31,59.3,16.3,61.34969325 +minicpm-1b-sft,INT4-MIXED,31,58.3,16.4,60.97560976 +minicpm-1b-sft,INT4-MIXED,1014,122.3,16.6,60.24096386 +minicpm-1b-sft,INT4-MIXED,1014,127.1,16.7,59.88023952 +exaone-4.0-1.2b,INT8-CW,32,46.8,17,58.82352941 +qwen3.5-0.8b,INT4-MIXED,1024,236.9,17.2,58.13953488 +lfm2-8b-a1b,INT8-CW,32,97.4,17.3,57.80346821 +qwen3.5-0.8b,INT4-MIXED,1024,238.9,17.4,57.47126437 +qwen3.5-0.8b,INT4-MIXED,85,106.7,17.5,57.14285714 +lfm2-8b-a1b,INT8-CW,1024,315,18,55.55555556 +qwen3.5-0.8b,INT8-CW,1024,248,18,55.55555556 +exaone-4.0-1.2b,INT8-CW,1024,141.5,18.1,55.24861878 +qwen3.5-0.8b,INT4-MIXED,85,118.9,18.1,55.24861878 +qwen3.5-0.8b,INT8-CW,85,116.7,18.2,54.94505495 +qwen2.5-1.5b-instruct,INT8-CW,32,38.1,18.3,54.64480874 +qwen2.5-coder-1.5b-instruct,INT8-CW,32,37,18.3,54.64480874 +deepseek-r1-distill-qwen-1.5b,INT8-CW,32,37.8,18.4,54.34782609 +qwen2.5-1.5b-instruct,INT8-CW,32,38.6,18.4,54.34782609 +lfm2-24b-a2b,INT4-MIXED,1024,727.6,18.7,53.47593583 +lfm2-24b-a2b,INT4-MIXED,32,228.4,18.7,53.47593583 +phi-2,INT4-MIXED,32,59.5,18.7,53.47593583 +qwen3.5-0.8b,INT4-MIXED,1024,254.2,18.7,53.47593583 +minicpm-1b-sft,INT4-MIXED,31,64.5,18.8,53.19148936 +stable-zephyr-3b-dpo,INT4-MIXED,32,59.6,18.8,53.19148936 +minicpm-1b-sft,INT8-CW,31,66.7,18.9,52.91005291 +qwen3.5-0.8b,INT4-MIXED,85,106.5,18.9,52.91005291 +minicpm-1b-sft,INT4-MIXED,1014,148.2,19,52.63157895 +deepseek-r1-distill-qwen-1.5b,INT8-CW,1024,139.2,19.1,52.35602094 +qwen2.5-coder-1.5b-instruct,INT8-CW,1024,138.7,19.1,52.35602094 +qwen2.5-1.5b-instruct,INT8-CW,1024,138.8,19.2,52.08333333 +qwen2.5-1.5b-instruct,INT8-CW,1024,138.6,19.2,52.08333333 +smollm3-3b,INT4-MIXED,32,60.1,20.2,49.5049505 +phi-2,INT4-MIXED,32,61.7,20.3,49.26108374 +minicpm-1b-sft,INT8-CW,1014,139.6,20.4,49.01960784 +minicpm5-1b,FP16,32,28.5,20.7,48.30917874 +phi-2,INT4-MIXED,1024,286.8,21,47.61904762 +smollm3-3b,INT4-MIXED,32,51.3,21,47.61904762 +stable-zephyr-3b-dpo,INT4-MIXED,1024,281,21,47.61904762 +stable-zephyr-3b-dpo,INT4-MIXED,32,66,21.1,47.39336493 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,62.1,21.5,46.51162791 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,54.6,21.5,46.51162791 +smollm3-3b,INT4-MIXED,32,53.5,21.5,46.51162791 +smollm3-3b,INT4-MIXED,1024,222.5,21.5,46.51162791 +minicpm5-1b,FP16,1024,86.1,21.6,46.2962963 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,179.7,21.6,46.2962963 +llama-3.2-3b-instruct,INT4-MIXED,32,42.1,21.7,46.08294931 +qwen2.5-1.5b-instruct,INT4-MIXED,32,75.6,22.1,45.24886878 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,188.2,22.2,45.04504505 +qwen2.5-1.5b-instruct,INT4-MIXED,32,73.2,22.3,44.84304933 +smollm3-3b,INT4-MIXED,1024,233.9,22.4,44.64285714 +smollm3-3b,INT4-MIXED,1024,265.3,22.6,44.24778761 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,57.3,22.7,44.05286344 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,223.3,22.7,44.05286344 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,228.9,22.7,44.05286344 +llama-3.2-3b-instruct,INT4-MIXED,32,43.4,22.8,43.85964912 +phi-2,INT4-MIXED,1024,361.2,22.8,43.85964912 +llama-3.2-3b-instruct,INT4-MIXED,32,44.7,22.9,43.66812227 +llama-3.2-3b-instruct,INT4-MIXED,1024,228.1,23.3,42.91845494 +stable-zephyr-3b-dpo,INT4-MIXED,1024,314.1,23.4,42.73504274 +gemma-3-1b-it,FP16,32,37.6,23.7,42.19409283 +qwen3.5-0.8b,FP16,85,91.6,23.9,41.84100418 +qwen3.5-0.8b,FP16,1024,249.6,24,41.66666667 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,419,24.1,41.49377593 +gemma-3-1b-it,FP16,1024,103.7,24.2,41.32231405 +llama-3.2-3b-instruct,INT4-MIXED,1024,243.1,24.3,41.15226337 +llama-3.2-3b-instruct,INT4-MIXED,1024,265.5,24.5,40.81632653 +qwen2.5-1.5b-instruct,INT8-CW,32,79.1,25,40 +qwen2.5-1.5b-instruct,INT8-CW,1024,178.8,25.7,38.91050584 +tiny-llama-1.1b-chat,FP16,32,29.3,25.9,38.61003861 +gemma-4-e2b-it,INT4-MIXED,274,284.3,26.6,37.59398496 +qwen3-4b,INT4-MIXED,32,58.6,26.7,37.45318352 +tiny-llama-1.1b-chat,FP16,1024,106.7,26.7,37.45318352 +phi-4-mini-reasoning,INT4-MIXED,32,62.5,27.6,36.23188406 +phi-4-mini-instruct,INT4-MIXED,32,63.4,27.7,36.10108303 +qwen3-4b,INT4-MIXED,32,60.9,27.7,36.10108303 +gemma-4-e2b-it,INT4-MIXED,274,313.5,28,35.71428571 +gpt-oss-20b,INT4-MIXED,32,221.7,28.1,35.58718861 +lfm2.5-1.2b-instruct,FP16,32,33,28.2,35.46099291 +lfm2-1.2b,FP16,1024,140.1,28.3,35.33568905 +lfm2-1.2b,FP16,32,33.5,28.3,35.33568905 +lfm2.5-1.2b-instruct,FP16,1024,139.3,28.3,35.33568905 +afm-4.5b,INT4-MIXED,32,51.9,28.4,35.21126761 +phi-4-mini-instruct,INT4-MIXED,32,65.5,28.4,35.21126761 +phi-4-mini-reasoning,INT4-MIXED,32,63.5,28.8,34.72222222 +gemma-3-4b-it,INT4-MIXED,32,74.7,28.9,34.60207612 +phi-4-mini-reasoning,INT4-MIXED,32,65,28.9,34.60207612 +qwen3-4b,INT4-MIXED,1024,324.6,29.2,34.24657534 +phi-4-mini-reasoning,INT4-MIXED,1024,376.8,29.3,34.12969283 +phi-4-mini-instruct,INT4-MIXED,1024,355.7,29.5,33.89830508 +phi-3.5-vision-instruct,INT4-MIXED,802,491.2,29.6,33.78378378 +gemma-3-4b-it,INT4-MIXED,32,75.3,29.8,33.55704698 +gpt-oss-20b,INT4-MIXED,1024,648.2,29.8,33.55704698 +llama-3.2-1b-instruct,FP16,32,35,29.8,33.55704698 +gemma-3-4b-it,INT4-MIXED,32,76.6,30.1,33.22259136 +phi-4-mini-reasoning,INT4-MIXED,1024,368,30.2,33.11258278 +afm-4.5b,INT4-MIXED,1024,378.4,30.3,33.00330033 +phi-4-mini-instruct,INT4-MIXED,1024,378.1,30.3,33.00330033 +whisper-large-v3,INT4-MIXED,32,288.2,30.3,33.00330033 +qwen3-4b,INT4-MIXED,1024,345.6,30.4,32.89473684 +phi-4-mini-instruct,INT4-MIXED,32,68.1,30.5,32.78688525 +gemma-3-4b-it,INT4-MIXED,1024,355.4,30.6,32.67973856 +phi-3.5-vision-instruct,INT4-MIXED,1032,601,30.6,32.67973856 +llama-3.2-1b-instruct,FP16,1024,131.7,30.7,32.5732899 +phi-4-mini-reasoning,INT4-MIXED,1024,389.4,30.8,32.46753247 +phi-2,INT8-CW,32,67.3,31.5,31.74603175 +gemma-3-4b-it,INT4-MIXED,1024,397.4,31.7,31.54574132 +gemma-3-4b-it,INT4-MIXED,1024,374.7,31.8,31.44654088 +phi-4-mini-instruct,INT4-MIXED,1024,462,32.2,31.05590062 +gemma-4-e2b-it,INT4-MIXED,1024,463.3,32.7,30.58103976 +minicpm-1b-sft,FP16,31,61.7,33,30.3030303 +whisper-large-v3,INT4-MIXED,1024,361.1,33.3,30.03003003 +whisper-large-v3,INT8-CW,32,314.9,33.3,30.03003003 +deepseek-r1-distill-qwen-1.5b,FP16,32,41.1,33.5,29.85074627 +qwen2.5-1.5b-instruct,FP16,32,40.8,33.6,29.76190476 +qwen2.5-coder-1.5b-instruct,FP16,32,40.4,33.6,29.76190476 +whisper-large-v3,INT8-CW,1024,364.7,33.8,29.58579882 +gemma-4-e2b-it,INT4-MIXED,1024,496,33.9,29.49852507 +phi-2,INT8-CW,1024,325,34,29.41176471 +smollm3-3b,INT8-CW,32,57.1,34,29.41176471 +qwen2.5-coder-3b-instruct,INT8-CW,32,58.2,34.1,29.3255132 +deepseek-r1-distill-qwen-1.5b,FP16,1024,166.6,34.4,29.06976744 +exaone-4.0-1.2b,FP16,32,41.9,34.4,29.06976744 +minicpm-1b-sft,FP16,1014,169.4,34.5,28.98550725 +qwen2.5-1.5b-instruct,FP16,1024,166.8,34.6,28.9017341 +qwen2.5-coder-1.5b-instruct,FP16,1024,162.4,34.7,28.8184438 +gpt-oss-20b,INT4-MIXED,32,215.6,35.2,28.40909091 +phi-4-multimodal-instruct,INT4-MIXED,786,540.4,35.2,28.40909091 +smollm3-3b,INT8-CW,1024,277.4,35.2,28.40909091 +phi-4-multimodal-instruct,INT4-MIXED,578,456.9,35.3,28.3286119 +exaone-4.0-1.2b,FP16,1024,330.3,35.4,28.24858757 +qwen2.5-coder-3b-instruct,INT8-CW,1024,268.4,35.5,28.16901408 +gemma-4-e2b-it,INT8-CW,274,317.9,35.8,27.93296089 +phi-4-multimodal-instruct,INT4-MIXED,1362,1124.1,36.3,27.54820937 +whisper-large-v3,FP16,32,269.7,36.3,27.54820937 +codellama-7b,INT4-MIXED,32,52.6,36.6,27.32240437 +llama-2-7b-chat-hf,INT4-MIXED,32,53.1,36.6,27.32240437 +phi-4-multimodal-instruct,INT4-MIXED,1570,1240.6,36.6,27.32240437 +gpt-oss-20b,INT4-MIXED,1024,682.1,36.7,27.2479564 +whisper-large-v3,FP16,1024,330,37.1,26.9541779 +llama-3.2-3b-instruct,INT8-CW,32,47,37.8,26.45502646 +llama-2-7b-chat-hf,INT4-MIXED,32,56.2,38.7,25.83979328 +biomistral-7b-slerp,INT4-MIXED,7,44.9,38.8,25.77319588 +mistral-7b-instruct-v0.2,INT4-MIXED,32,54.2,38.9,25.70694087 +zephyr-7b-beta,INT4-MIXED,32,53.8,38.9,25.70694087 +mistral-7b-instruct-v0.3,INT4-MIXED,32,55,39,25.64102564 +codellama-7b,INT4-MIXED,32,55.7,39.1,25.57544757 +llama-3.2-3b-instruct,INT8-CW,1024,272,39.1,25.57544757 +neural-chat-7b-v3-3,INT4-MIXED,32,53.6,39.1,25.57544757 +llama-2-7b-chat-hf,INT4-MIXED,1024,493.8,39.4,25.38071066 +codellama-7b,INT4-MIXED,1024,486.2,39.6,25.25252525 +minicpm3-4b,INT4-MIXED,32,171.8,40.2,24.87562189 +falcon-7b-instruct,INT4-MIXED,32,56.9,40.5,24.69135802 +minicpm3-4b,INT4-MIXED,32,176,40.5,24.69135802 +phi-4-multimodal-instruct,INT4-MIXED,578,526.6,40.6,24.63054187 +minicpm3-4b,INT4-MIXED,32,176.6,40.7,24.57002457 +qwen3-vl-4b-instruct,INT4-MIXED,4937,6463.6,40.7,24.57002457 +phi-4-multimodal-instruct,INT4-MIXED,786,637.4,40.9,24.44987775 +qwen3-vl-4b-instruct,INT4-MIXED,4907,6197.7,40.9,24.44987775 +mistral-7b-instruct-v0.3,INT4-MIXED,32,60.8,41.1,24.33090024 +mistral-7b-instruct-v0.2,INT4-MIXED,32,59.4,41.2,24.27184466 +zephyr-7b-beta,INT4-MIXED,1024,506.8,41.2,24.27184466 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,510.8,41.3,24.21307506 +biomistral-7b-slerp,INT4-MIXED,7,48.3,41.4,24.15458937 +llama-2-7b-chat-hf,INT4-MIXED,1024,533.9,41.4,24.15458937 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,512,41.4,24.15458937 +neural-chat-7b-v3-3,INT4-MIXED,32,58.4,41.4,24.15458937 +mistral-7b-instruct-v0.1,INT4-MIXED,32,60.9,41.5,24.09638554 +minicpm3-4b,INT4-MIXED,1024,727.4,41.6,24.03846154 +qwen3-vl-4b-instruct,INT4-MIXED,4907,6260.5,41.6,24.03846154 +mistral-7b-instruct-v0.3,INT4-MIXED,32,59.1,41.7,23.98081535 +qwen3-vl-4b-instruct,INT4-MIXED,4937,6553,41.7,23.98081535 +mistral-7b-instruct-v0.2,INT4-MIXED,32,59.6,41.8,23.92344498 +neural-chat-7b-v3-3,INT4-MIXED,1024,507.2,41.8,23.92344498 +qwen3-vl-4b-instruct,INT4-MIXED,4907,6485.5,41.8,23.92344498 +codellama-7b,INT4-MIXED,1024,587.4,41.9,23.86634845 +qwen2.5-7b-instruct-1m,INT4-MIXED,32,54.7,42,23.80952381 +qwen3-vl-4b-instruct,INT4-MIXED,4937,6781.2,42,23.80952381 +qwen2.5-7b-instruct,INT4-MIXED,32,56.5,42.2,23.69668246 +qwen2.5-7b-instruct,INT4-MIXED,32,56.8,42.3,23.64066194 +phi-4-multimodal-instruct,INT4-MIXED,1570,1448.8,42.4,23.58490566 +stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,43.7,42.4,23.58490566 +minicpm3-4b,INT4-MIXED,1024,825.3,42.6,23.4741784 +phi-4-multimodal-instruct,INT4-MIXED,1362,1303.4,42.6,23.4741784 +stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,44.9,42.6,23.4741784 +falcon-7b-instruct,INT4-MIXED,1024,527.7,42.8,23.36448598 +llama-3-8b-instruct,INT4-MIXED,32,58.3,42.9,23.31002331 +deepseek-r1-distill-llama-8b,INT4-MIXED,35,65.7,43,23.25581395 +minicpm3-4b,INT4-MIXED,1024,780.7,43,23.25581395 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,79.8,43,23.25581395 +gemma-4-e2b-it,INT8-CW,1024,508.7,43.1,23.20185615 +minicpm-v-2_6,INT4-MIXED,228,546.7,43.2,23.14814815 +llama-3.1-8b-instruct,INT4-MIXED,32,59.6,43.4,23.04147465 +minicpm-o-2_6,INT4-MIXED,238,538.8,43.4,23.04147465 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,555.8,43.4,23.04147465 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,554.9,43.4,23.04147465 +deepseek-r1-distill-qwen-7b,INT4-MIXED,32,58.7,43.5,22.98850575 +qwen2-7b-instruct,INT4-MIXED,32,60.1,43.6,22.93577982 +neural-chat-7b-v3-3,INT4-MIXED,1024,625.4,43.7,22.88329519 +qwen2.5-7b-instruct,INT4-MIXED,32,59.8,43.7,22.88329519 +qwen2.5-7b-instruct-1m,INT4-MIXED,1024,458.7,43.7,22.88329519 +qwen2.5-7b-instruct,INT4-MIXED,1024,460.2,43.8,22.83105023 +mistral-7b-instruct-v0.1,INT4-MIXED,1025,670.7,44,22.72727273 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,630.2,44,22.72727273 +qwen2-7b-instruct,INT4-MIXED,32,61.4,44,22.72727273 +qwen2.5-7b-instruct,INT4-MIXED,32,60.3,44,22.72727273 +qwen2.5-7b-instruct,INT4-MIXED,32,59.2,44,22.72727273 +deepseek-r1-distill-qwen-7b,INT4-MIXED,32,59.8,44.1,22.67573696 +qwen2.5-7b-instruct,INT4-MIXED,32,60.2,44.1,22.67573696 +qwen2.5-7b-instruct,INT4-MIXED,1024,464.5,44.1,22.67573696 +qwen2.5-7b-instruct-1m,INT4-MIXED,32,60.7,44.1,22.67573696 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,641.6,44.3,22.57336343 +bloomz-7b1,INT4-MIXED,32,60.9,44.5,22.47191011 +qwen3-8b,INT4-MIXED,32,68.5,44.5,22.47191011 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,543.5,44.6,22.42152466 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,80.9,44.8,22.32142857 +llama-3-8b-instruct,INT4-MIXED,32,63,44.9,22.27171492 +llama-3-8b-instruct,INT4-MIXED,32,63.3,44.9,22.27171492 +minicpm-v-2_6,INT4-MIXED,228,569.7,44.9,22.27171492 +falcon-7b-instruct,INT4-MIXED,32,60.8,45,22.22222222 +gemma-3-4b-it,INT8-CW,32,81.4,45,22.22222222 +llama-3.1-8b-instruct,INT4-MIXED,32,63.5,45,22.22222222 +phi-4-mini-reasoning,INT8-CW,32,68.5,45,22.22222222 +fara-7b,INT4-MIXED,32,111.6,45.1,22.172949 +minicpm-v-2_6,INT4-MIXED,228,578.7,45.1,22.172949 +minicpm4-8b,INT4-MIXED,32,65.4,45.1,22.172949 +deepseek-r1-distill-llama-8b,INT4-MIXED,35,67.2,45.3,22.07505519 +deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,505.1,45.3,22.07505519 +llama-3-8b-instruct,INT4-MIXED,1024,515.2,45.3,22.07505519 +qwen2-7b-instruct,INT4-MIXED,1024,501.3,45.4,22.02643172 +qwen2.5-7b-instruct,INT4-MIXED,1024,500.9,45.4,22.02643172 +deepseek-r1-distill-llama-8b,INT4-MIXED,1135,581.7,45.5,21.97802198 +minicpm-o-2_6,INT4-MIXED,238,566.6,45.5,21.97802198 +llama-3-8b-instruct,INT4-MIXED,32,63.2,45.6,21.92982456 +qwen2.5-7b-instruct,INT4-MIXED,1024,501.2,45.6,21.92982456 +minicpm-v-4_5,INT4-MIXED,217,569.5,45.7,21.88183807 +deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,572.8,45.8,21.83406114 +phi-4-mini-instruct,INT8-CW,32,67.2,45.8,21.83406114 +qwen2-7b-instruct,INT4-MIXED,1024,572.1,45.8,21.83406114 +qwen2.5-7b-instruct,INT4-MIXED,1024,574.7,45.8,21.83406114 +qwen2.5-7b-instruct,INT4-MIXED,1024,574.1,45.9,21.78649237 +llama-3.1-8b-instruct,INT4-MIXED,1024,518.4,46,21.73913043 +qwen2.5-7b-instruct-1m,INT4-MIXED,1024,578.3,46.1,21.69197397 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,599.7,46.2,21.64502165 +qwen3-4b,INT8-CW,32,65.4,46.2,21.64502165 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,82.8,46.3,21.59827214 +qwen3-8b,INT4-MIXED,32,67.9,46.5,21.50537634 +minicpm4-8b,INT4-MIXED,1024,553.3,46.7,21.41327623 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,677.6,46.7,21.41327623 +bloomz-7b1,INT4-MIXED,1024,677.4,46.8,21.36752137 +fara-7b,INT4-MIXED,1024,786.1,46.8,21.36752137 +phi-4-mini-reasoning,INT8-CW,1024,402.4,46.8,21.36752137 +bloomz-7b1,INT4-MIXED,32,66.9,47.1,21.23142251 +gemma-3-4b-it,INT8-CW,1024,407.2,47.1,21.23142251 +minicpm4-8b,INT4-MIXED,32,71.9,47.1,21.23142251 +qwen3-8b,INT4-MIXED,1024,545.4,47.1,21.23142251 +afm-4.5b,INT8-CW,32,60.4,47.2,21.18644068 +qwen3-8b,INT4-MIXED,32,70.1,47.2,21.18644068 +falcon-7b-instruct,INT4-MIXED,1024,637,47.3,21.14164905 +llama-3-8b-instruct,INT4-MIXED,1024,557.9,47.3,21.14164905 +llama-3-8b-instruct,INT4-MIXED,1024,562,47.4,21.09704641 +phi-3.5-vision-instruct,INT8-CW,802,503.6,47.5,21.05263158 +deepseek-r1-distill-llama-8b,INT4-MIXED,1135,636.1,47.6,21.00840336 +phi-4-mini-instruct,INT8-CW,1024,410.3,47.6,21.00840336 +qwen3_8b_draft_eagle3,INT4-MIXED,32,65.4,47.6,21.00840336 +minicpm4-8b,INT4-MIXED,32,71.4,47.7,20.96436059 +llama-3.1-8b-instruct,INT4-MIXED,1024,565,47.9,20.87682672 +llama-3-8b-instruct,INT4-MIXED,1024,640,48,20.83333333 +minicpm-v-4_5,INT4-MIXED,217,587.4,48.5,20.6185567 +phi-3.5-vision-instruct,INT8-CW,1032,620.2,48.5,20.6185567 +qwen3-4b,INT8-CW,1024,375.9,48.9,20.44989775 +minicpm4-8b,INT4-MIXED,1024,602.7,49.1,20.36659878 +afm-4.5b,INT8-CW,1024,372.1,49.2,20.32520325 +bloomz-7b1,INT4-MIXED,1024,786.4,49.2,20.32520325 +qwen3-8b,INT4-MIXED,1024,575.8,49.2,20.32520325 +qwen3-8b,INT4-MIXED,1024,654.9,49.8,20.08032129 +zephyr-7b-beta,INT4-MIXED,32,65.8,49.9,20.04008016 +minicpm4-8b,INT4-MIXED,1024,688,50.2,19.92031873 +gemma-7b-it,INT4-MIXED,32,74.8,51,19.60784314 +zephyr-7b-beta,INT4-MIXED,1024,547,52.3,19.12045889 +qwen3_8b_draft_eagle3,INT4-MIXED,1024,617.5,53.3,18.76172608 +gemma-7b-it,INT4-MIXED,32,79.8,53.7,18.62197393 +gemma-7b-it,INT4-MIXED,1024,623.1,53.8,18.58736059 +phi-4-multimodal-instruct,INT8-CW,578,501.3,54.4,18.38235294 +phi-4-multimodal-instruct,INT8-CW,786,603.3,54.8,18.24817518 +qwen3.5-9b,INT4-MIXED,83,203.5,55.1,18.14882033 +ltx-video,INT4-MIXED,11,56.1,55.5,18.01801802 +qwen3.5-9b,INT4-MIXED,1024,956.8,55.5,18.01801802 +phi-4-multimodal-instruct,INT8-CW,1570,1361.9,56,17.85714286 +minicpm3-4b,INT8-CW,32,193.7,56.3,17.76198934 +deepseek-r1-distill-llama-8b,INT4-MIXED,35,75.8,56.4,17.73049645 +gemma-7b-it,INT4-MIXED,1024,757.9,56.4,17.73049645 +llama-3.1-8b-instruct,INT4-MIXED,32,74.5,56.5,17.69911504 +ltx-video,INT8-CW,11,57,56.5,17.69911504 +phi-4-multimodal-instruct,INT8-CW,1362,1248.4,56.9,17.57469244 +phi-2,FP16,32,64.9,57.3,17.45200698 +gemma-4-e2b-it,FP16,274,348.7,57.6,17.36111111 +qwen3.5-9b,INT4-MIXED,83,218.2,57.8,17.30103806 +gemma-2-9b-it,INT4-MIXED,32,78.8,58.3,17.15265866 +qwen3.5-9b,INT4-MIXED,1024,1143.4,58.7,17.03577513 +deepseek-r1-distill-llama-8b,INT4-MIXED,1135,760.8,58.8,17.00680272 +gemma-2-9b-it,INT4-MIXED,32,82.2,59.5,16.80672269 +llama-3.1-8b-instruct,INT4-MIXED,1024,727.5,59.8,16.72240803 +gemma-2-9b-it,INT4-MIXED,32,83.5,60,16.66666667 +phi-2,FP16,1024,415.6,61.1,16.36661211 +gemma-2-9b-it,INT4-MIXED,1024,687.2,61.6,16.23376623 +minicpm3-4b,INT8-CW,1024,901.2,61.6,16.23376623 +gemma-2-9b-it,INT4-MIXED,1024,723.3,63.1,15.84786054 +gemma-2-9b-it,INT4-MIXED,1024,788.1,63.8,15.67398119 +stable-diffusion-xl-1.0-inpainting-0.1,FP16,32,66.5,64.3,15.55209953 +gemma-4-e2b-it,FP16,1024,580,65.4,15.29051988 +lcm-dreamshaper-v7,INT8-HYBRID,1024,67.8,66.1,15.12859304 +lcm-dreamshaper-v7,INT8-HYBRID,32,68.7,66.3,15.08295626 +llama-3.2-3b-instruct,FP16,32,72.7,66.7,14.99250375 +videochat-flash-qwen2_5-7b_internvideo2-1b,INT4-MIXED,14531,84815.8,67.6,14.79289941 +dolly-v2-12b,INT4-MIXED,32,102.2,67.8,14.74926254 +llama-3.2-3b-instruct,FP16,1024,384.5,68.8,14.53488372 +llama-2-13b-chat-hf,INT4-MIXED,32,100.5,69.7,14.3472023 +dolly-v2-12b,INT4-MIXED,1024,1253.8,72,13.88888889 +smollm3-3b,FP16,32,78.1,73,13.69863014 +gemma-3-12b-it,INT4-MIXED,32,128.5,73.2,13.66120219 +qwen2.5-coder-3b-instruct,FP16,32,78.2,73.9,13.53179973 +llama-2-13b-chat-hf,INT4-MIXED,1024,962,74,13.51351351 +qwen2.5-7b-instruct,INT8-CW,32,95,74.7,13.38688086 +qwen2.5-7b-instruct-1m,INT8-CW,32,94.6,74.8,13.36898396 +qwen2-7b-instruct,INT8-CW,32,94.9,74.9,13.35113485 +smollm3-3b,FP16,1024,308.1,75,13.33333333 +deepseek-r1-distill-qwen-7b,INT8-CW,32,95,75.3,13.28021248 +falcon-7b-instruct,INT8-CW,32,90.8,75.5,13.24503311 +qwen2.5-7b-instruct,INT8-CW,32,94.9,75.8,13.19261214 +fara-7b,INT8-CW,32,143.3,76,13.15789474 +qwen2.5-coder-3b-instruct,FP16,1024,309.3,76,13.15789474 +qwen2.5-vl-7b-instruct,INT8-CW,32,115.7,76.1,13.14060447 +codellama-7b,INT8-CW,32,88.7,76.2,13.12335958 +minicpm-o-2_6,INT8-CW,238,555.3,76.2,13.12335958 +qwen2-7b-instruct,INT8-CW,1024,565.8,76.3,13.1061599 +qwen2.5-7b-instruct-1m,INT8-CW,1024,565.5,76.3,13.1061599 +qwen2.5-7b-instruct,INT8-CW,1024,574.4,76.4,13.08900524 +gemma-3-12b-it,INT4-MIXED,32,134.5,76.9,13.00390117 +phi-3-mini-128k-instruct,FP16,32,84.6,77,12.98701299 +gemma-3-12b-it,INT4-MIXED,1024,1046.4,77.2,12.95336788 +minicpm-v-2_6,INT8-CW,228,552.6,77.3,12.93661061 +phi-3.5-mini-instruct,FP16,32,85.3,77.4,12.91989664 +qwen2.5-vl-7b-instruct,INT8-CW,1024,650.8,77.4,12.91989664 +phi-3-mini-4k-instruct,FP16,32,84.7,77.5,12.90322581 +qwen2.5-7b-instruct,INT8-CW,1024,563.6,77.5,12.90322581 +deepseek-r1-distill-qwen-7b,INT8-CW,1024,572.5,77.6,12.88659794 +falcon-7b-instruct,INT8-CW,1024,793,77.6,12.88659794 +fara-7b,INT8-CW,1024,786.1,77.6,12.88659794 +lcm-dreamshaper-v7,INT8-CW,1024,78.9,77.6,12.88659794 +phi-4,INT4-MIXED,32,112,77.6,12.88659794 +phi-4-reasoning,INT4-MIXED,32,112,77.6,12.88659794 +llama-2-7b-chat-hf,INT8-CW,32,88.6,78.4,12.75510204 +lcm-dreamshaper-v7,INT8-CW,32,80.5,78.8,12.69035533 +lcm-dreamshaper-v7,FP16,32,80.9,79.5,12.57861635 +codellama-7b,INT8-CW,1024,556.1,79.8,12.53132832 +lcm-dreamshaper-v7,FP16,1024,79.8,79.9,12.51564456 +lcm-dreamshaper-v7,INT8-CW,1024,82.3,80,12.5 +phi-4-mini-reasoning,FP16,32,89.4,80.3,12.45330012 +lcm-dreamshaper-v7,INT8-CW,32,79.4,80.4,12.43781095 +phi-4-mini-instruct,FP16,32,88.8,80.4,12.43781095 +neural-chat-7b-v3-3,INT8-CW,32,94.7,80.6,12.40694789 +zephyr-7b-beta,INT8-CW,32,95.5,80.7,12.39157373 +phi-3.5-vision-instruct,FP16,802,702.4,80.8,12.37623762 +gemma-3-12b-it,INT4-MIXED,1024,1194.5,81.1,12.33045623 +biomistral-7b-slerp,INT8-CW,7,85.9,81.3,12.300123 +phi-3-mini-128k-instruct,FP16,1024,553.8,81.3,12.300123 +phi-4,INT4-MIXED,1024,1094.3,81.3,12.300123 +qwen1.5-14b-chat,INT4-MIXED,32,115.9,81.3,12.300123 +qwen3_8b_draft_eagle3,INT8-CW,1024,635.1,81.3,12.300123 +mistral-7b-instruct-v0.2,INT8-CW,32,93.8,81.4,12.28501229 +mistral-7b-instruct-v0.1,INT8-CW,32,93.8,81.5,12.26993865 +phi-3-mini-4k-instruct,FP16,1024,562.4,81.6,12.25490196 +phi-3.5-mini-instruct,FP16,1024,552.1,81.8,12.22493888 +phi-3.5-vision-instruct,FP16,1032,877.6,81.9,12.21001221 +phi-4-reasoning,INT4-MIXED,1024,1072.1,81.9,12.21001221 +mistral-7b-instruct-v0.3,INT8-CW,32,93.8,82,12.19512195 +llama-2-7b-chat-hf,INT8-CW,1024,566.1,82.2,12.16545012 +phi-4-mini-instruct,FP16,1024,539.6,82.3,12.15066829 +phi-4-reasoning,INT4-MIXED,32,123.5,82.3,12.15066829 +phi-4-mini-reasoning,FP16,1024,539.1,82.6,12.10653753 +phi-4,INT4-MIXED,32,125,82.8,12.07729469 +neural-chat-7b-v3-3,INT8-CW,1024,590.9,83,12.04819277 +deepseek-r1-distill-qwen-14b,INT4-MIXED,32,114.3,83.1,12.03369434 +qwen3_8b_draft_eagle3,INT8-CW,32,105.6,83.2,12.01923077 +zephyr-7b-beta,INT8-CW,1024,584,83.2,12.01923077 +gemma-3-4b-it,FP16,32,93.4,83.3,12.00480192 +bloomz-7b1,INT8-CW,32,96.3,83.9,11.91895113 +mistral-7b-instruct-v0.1,INT8-CW,1025,634.5,83.9,11.91895113 +mistral-7b-instruct-v0.2,INT8-CW,1024,586.8,84,11.9047619 +gemma-3-4b-it,FP16,1024,515.8,84.1,11.89060642 +mistral-7b-instruct-v0.3,INT8-CW,1024,590.4,84.2,11.87648456 +qwen1.5-14b-chat,INT4-MIXED,1024,1152.9,84.2,11.87648456 +deepseek-r1-distill-llama-8b,INT8-CW,35,108.6,84.9,11.77856302 +llama-3.1-8b-instruct,INT8-CW,32,99,85,11.76470588 +phi-4-reasoning,INT4-MIXED,1024,1177.6,85.8,11.65501166 +phi-4-reasoning,INT4-MIXED,32,124.2,86.2,11.60092807 +phi-4,INT4-MIXED,1024,1301.1,86.4,11.57407407 +qwen3-4b,FP16,32,93.5,86.6,11.54734411 +llama-3-8b-instruct,INT8-CW,32,100.1,86.7,11.53402537 +llama-3.1-8b-instruct,INT8-CW,1024,600.5,87.2,11.46788991 +bloomz-7b1,INT8-CW,1024,792.2,87.4,11.4416476 +deepseek-r1-distill-qwen-14b,INT4-MIXED,1024,1175.1,87.5,11.42857143 +llama-3-8b-instruct,INT8-CW,1024,610.4,88.3,11.32502831 +llava-v1.6-mistral-7b-hf,INT8-CW,2944,2875.9,88.3,11.32502831 +deepseek-r1-distill-llama-8b,INT8-CW,1135,663.6,88.5,11.29943503 +llama-2-13b-chat-hf,INT4-MIXED,32,121.3,88.9,11.24859393 +starcoder,INT4-MIXED,32,139.1,89.5,11.17318436 +phi-4-reasoning,INT4-MIXED,1024,1347.6,90.1,11.09877913 +afm-4.5b,FP16,32,99.5,90.3,11.07419712 +qwen3-4b,FP16,1024,476.6,90.3,11.07419712 +qwen3-8b,INT8-CW,32,102.1,91.8,10.89324619 +minicpm-v-4_5,INT8-CW,217,559.8,92.2,10.84598698 +afm-4.5b,FP16,1024,483.9,92.4,10.82251082 +llama-2-13b-chat-hf,INT4-MIXED,1024,1172.7,93.3,10.71811361 +starcoder,INT4-MIXED,1024,1388.6,93.6,10.68376068 +gemma-7b-it,INT8-CW,32,115,93.9,10.64962726 +qwen3-8b,INT8-CW,1024,644.9,94,10.63829787 +minicpm4-8b,INT8-CW,32,107.2,94.5,10.58201058 +minicpm3-4b,FP16,32,171.3,94.8,10.54852321 +phi-4-multimodal-instruct,FP16,786,764.1,95.1,10.51524711 +phi-4-multimodal-instruct,FP16,578,637.4,95.6,10.46025105 +phi-4-multimodal-instruct,FP16,1362,1535.3,96.2,10.3950104 +minicpm4-8b,INT8-CW,1024,698.9,96.6,10.35196687 +starcoder2-15b,INT4-MIXED,32,146.2,97.5,10.25641026 +gemma-7b-it,INT8-CW,1024,744.1,98.2,10.18329939 +phi-4-multimodal-instruct,FP16,1570,1678.8,98.7,10.13171226 \ No newline at end of file diff --git a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-368H.csv b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-368H.csv index 26abcbe8b00369..6b03fa8c977467 100644 --- a/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-368H.csv +++ b/docs/sphinx_setup/_static/benchmarks_files/llm_models_7-368H.csv @@ -1,345 +1,625 @@ Topology,Precision,Input Size,1st latency (ms),2nd latency (ms),2nd token per sec -llama-3.2-1b-instruct,INT4-MIXED,32,13,8,76.92 -llama-3.2-1b-instruct,INT4-MIXED,32,13.3,8,75.19 -minicpm4-0.5b,FP4-NORMALIZED,32,14.1,8.7,70.92 -gemma-3-270m,FP16,32,14.8,6.5,67.57 -minicpm4-0.5b,FP16,32,14.8,9.4,67.57 -llama-3.2-1b-instruct,INT8-CW,32,15,12.7,66.67 -minicpm4-0.5b,INT4-MIXED,32,15.4,3.9,64.94 -minicpm4-0.5b,INT4-MIXED,32,15.4,3.9,64.94 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,15.8,4.4,63.29 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,15.9,4.5,62.89 -qwen2.5-coder-0.5b-instruct,FP16,32,16.1,10.5,62.11 -minicpm4-0.5b,INT4-MIXED,32,16.4,4,60.98 -gemma-3-270m,INT8-CW,32,16.7,7.5,59.88 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,16.8,4.5,59.52 -minicpm4-0.5b,INT8-CW,32,17.3,5.5,57.80 -gemma-3-270m,INT4-MIXED,32,17.4,5.7,57.47 -qwen2.5-coder-0.5b-instruct,INT8-CW,32,17.8,6.1,56.18 -gemma-3-270m,FP16,1024,18.7,6.7,53.48 -gemma-2b-it,INT4-MIXED,32,19.1,15,52.36 -gemma-3-270m,INT4-MIXED,1024,19.1,4.3,52.36 -qwen3-embedding-0.6b,FP16,32,19.1,-1,52.36 -qwen3-embedding-0.6b,INT4-MIXED,32,19.6,-1,51.02 -gemma-2b-it,INT4-MIXED,32,19.8,15.7,50.51 -gemma-3-270m,INT8-CW,1024,19.9,4.6,50.25 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,19.9,9.8,50.25 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,19.9,10,50.25 -qwen2.5-1.5b-instruct,INT4-MIXED,32,20,9.9,50.00 -qwen3-embedding-0.6b,INT4-MIXED,32,20,-1,50.00 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,20.2,9.9,49.50 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,20.7,10.1,48.31 -qwen3-embedding-0.6b,INT4-MIXED,32,20.7,-1,48.31 -qwen3-embedding-0.6b,INT8-CW,32,21.3,-1,46.95 -qwen3-reranker-0.6b-seq-cls,FP16,22,21.3,-1,46.95 -gemma-3-1b-it,FP4-NORMALIZED,32,21.4,18.3,46.73 -gemma-3-1b-it,INT4-MIXED,32,21.7,11.2,46.08 -gemma-3-1b-it,INT4-MIXED,32,22.1,11.3,45.25 -gemma-3-1b-it,INT4-MIXED,32,22.4,11.7,44.64 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,22.5,12.6,44.44 -qwen2.5-1.5b-instruct,INT4-MIXED,32,22.5,10.8,44.44 -qwen2.5-1.5b-instruct,INT8-CW,32,23.2,15.7,43.10 -minicpm4-0.5b,INT4-MIXED,1024,23.6,4,42.37 -deepseek-r1-distill-qwen-1.5b,INT8-CW,32,23.9,15.7,41.84 -minicpm4-0.5b,INT4-MIXED,1024,24,4.2,41.67 -gemma-3-1b-it,FP16,32,24.3,20.8,41.15 -gemma-2-2b,INT4-MIXED,33,24.4,16.9,40.98 -gemma-3-1b-it,INT8-CW,32,24.5,13.2,40.82 -llama-3.2-1b-instruct,FP16,32,24.9,23.2,40.16 -qwen2.5-coder-1.5b-instruct,INT8-CW,32,25.2,15.7,39.68 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,26.1,4.5,38.31 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,26.2,4.6,38.17 -llama-3.2-3b-instruct,INT4-MIXED,32,26.4,18.1,37.88 -minicpm4-0.5b,INT8-CW,1024,26.4,5.9,37.88 -llama-3.2-3b-instruct,INT4-MIXED,32,26.8,18.6,37.31 -qwen3-reranker-0.6b-seq-cls,INT4-MIXED,22,26.8,-1,37.31 -qwen3-reranker-0.6b-seq-cls,INT4-MIXED,22,27.1,-1,36.90 -qwen3-reranker-0.6b-seq-cls,INT4-MIXED,22,27.6,-1,36.23 -llama-3.2-3b-instruct,INT4-MIXED,32,27.8,18.9,35.97 -gemma-2b-it,INT8-CW,32,28.3,25,35.34 -qwen2.5-coder-0.5b-instruct,INT8-CW,1024,28.6,6.3,34.97 -minicpm4-0.5b,INT4-MIXED,1024,28.9,4.1,34.60 -phi-3-mini-128k-instruct,INT4-MIXED,32,28.9,19.9,34.60 -phi-3-mini-4k-instruct,INT4-MIXED,32,28.9,19.8,34.60 -phi-3.5-mini-instruct,INT4-MIXED,32,28.9,19.8,34.60 -phi-3.5-mini-instruct,INT4-MIXED,32,29.1,20.4,34.36 -phi-3-mini-4k-instruct,INT4-MIXED,32,29.3,20.4,34.13 -gemma-2-2b,INT4-MIXED,33,29.6,17.5,33.78 -phi-3-mini-128k-instruct,INT4-MIXED,32,30,20.9,33.33 -deepseek-r1-distill-qwen-1.5b,FP4-NORMALIZED,32,30.3,27.5,33.00 -phi-3-mini-4k-instruct,INT4-MIXED,32,30.7,21.7,32.57 -stable-zephyr-3b-dpo,INT4-MIXED,32,31,15.7,32.26 -minicpm4-0.5b,FP4-NORMALIZED,1024,31.2,8.9,32.05 -qwen3-reranker-0.6b,INT4-MIXED,22,31.3,-1,31.95 -qwen3-reranker-0.6b-seq-cls,INT8-CW,22,31.4,-1,31.85 -qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,31.5,4.6,31.75 -qwen3-reranker-0.6b,INT4-MIXED,22,31.6,-1,31.65 -afm-4.5b,INT4-MIXED,32,31.9,24,31.35 -qwen2.5-1.5b-instruct,FP16,32,31.9,29.7,31.35 -qwen2.5-coder-1.5b-instruct,FP16,32,32,29.6,31.25 -deepseek-r1-distill-qwen-1.5b,FP16,32,32.1,29.6,31.15 -minicpm4-0.5b,FP16,1024,32.2,9.7,31.06 -phi-3.5-mini-instruct,INT4-MIXED,32,32.2,21.9,31.06 -qwen2.5-1.5b-instruct,FP4-NORMALIZED,32,32.3,27.5,30.96 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,32.5,18.7,30.77 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,32.5,18.1,30.77 -glm-edge-1.5b-chat,INT4-MIXED,32,32.6,11.1,30.67 -gemma-2-2b,INT8-CW,33,32.7,26,30.58 -qwen3-reranker-0.6b,FP16,22,32.7,-1,30.58 -minicpm-1b-sft,INT4-MIXED,31,33.7,9.6,29.67 -chatglm3-6b,INT4-MIXED,32,34.3,29.8,29.15 -qwen3-reranker-0.6b,INT4-MIXED,22,34.5,-1,28.99 -stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,34.5,34,28.99 -minicpm-1b-sft,INT4-MIXED,31,34.6,9.8,28.90 -glm-edge-1.5b-chat,INT8-CW,32,34.8,16.2,28.74 -stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,35.3,35.1,28.33 -qwen2.5-coder-3b-instruct,INT8-CW,32,35.6,30.5,28.09 -qwen2.5-coder-0.5b-instruct,FP16,1024,35.7,10.7,28.01 -llama-3.2-3b-instruct,INT8-CW,32,35.9,32.3,27.86 -minicpm-1b-sft,INT4-MIXED,31,36.2,10.5,27.62 -minicpm-1b-sft,FP16,31,36.3,27.9,27.55 -biomistral-7b-slerp,INT4-MIXED,7,36.4,34.5,27.47 -stablelm-3b-4e1t,INT4-MIXED,32,36.4,15.8,27.47 -qwen2.5-coder-3b-instruct,INT4-MIXED,32,36.5,19.1,27.40 -glm-edge-1.5b-chat,FP16,32,36.8,28.7,27.17 -qwen3-4b,INT4-MIXED,32,36.8,22.7,27.17 -phi-2,INT4-MIXED,32,36.9,15.9,27.10 -chatglm3-6b,INT4-MIXED,32,37,31.1,27.03 -ltx-video,INT4-MIXED,11,37,36.4,27.03 -codellama-7b,INT4-MIXED,32,37.4,32.8,26.74 -llama-2-7b-chat-hf,INT4-MIXED,32,37.4,32.8,26.74 -qwen3-4b,INT4-MIXED,32,37.6,23.4,26.60 -phi-2,INT4-MIXED,32,38.1,16.9,26.25 -biomistral-7b-slerp,INT4-MIXED,7,38.4,36.2,26.04 -ltx-video,INT8-CW,11,38.7,38,25.84 -qwen3-reranker-0.6b,INT8-CW,22,39.4,-1,25.38 -phi-2,INT8-CW,32,39.6,27.1,25.25 -stable-zephyr-3b-dpo,INT8-CW,32,39.6,27.2,25.25 -stablelm-3b-4e1t,INT8-CW,32,39.6,27.3,25.25 -lcm-dreamshaper-v7,INT8-HYBRID,32,39.8,38.6,25.13 -falcon-7b-instruct,INT4-MIXED,32,40.3,34.6,24.81 -mistral-7b-instruct-v0.3,INT4-MIXED,32,40.4,34.7,24.75 -phi-4-mini-reasoning,INT4-MIXED,32,40.6,22.9,24.63 -mistral-7b-instruct-v0.2,INT4-MIXED,32,40.7,34.6,24.57 -phi-4-mini-reasoning,INT4-MIXED,32,40.7,23.5,24.57 -lcm-dreamshaper-v7,INT8-HYBRID,1024,40.8,38.8,24.51 -llama-2-7b-chat-hf,INT4-MIXED,32,40.8,33.8,24.51 -phi-4-mini-instruct,INT4-MIXED,32,41,22.9,24.39 -minicpm-1b-sft,INT8-CW,31,41.2,15.6,24.27 -codellama-7b,INT4-MIXED,32,41.4,34.3,24.15 -phi-4-mini-instruct,INT4-MIXED,32,41.5,23.5,24.10 -qwen3-embedding-0.6b,INT4-MIXED,1024,41.5,-1,24.10 -qwen3-reranker-0.6b-seq-cls,INT4-MIXED,836,41.8,-1,23.92 -stable-zephyr-3b-dpo,INT4-MIXED,32,41.9,17.6,23.87 -stablelm-3b-4e1t,INT4-MIXED,32,41.9,19.2,23.87 -qwen3-embedding-0.6b,INT4-MIXED,1024,42.1,-1,23.75 -qwen2.5-7b-instruct-1m,INT4-MIXED,32,42.2,36.3,23.70 -phi-3-mini-4k-instruct,INT8-CW,32,42.8,37.9,23.36 -phi-3-mini-128k-instruct,INT8-CW,32,42.9,37.9,23.31 -phi-3.5-mini-instruct,INT8-CW,32,43,38.1,23.26 -phi-4-mini-reasoning,INT4-MIXED,32,43.2,23.8,23.15 -qwen3-reranker-0.6b-seq-cls,INT4-MIXED,836,43.2,-1,23.15 -llama-3.2-1b-instruct,INT4-MIXED,1024,43.4,8.4,23.04 -mistral-7b-instruct-v0.2,INT4-MIXED,32,43.6,35.8,22.94 -mistral-7b-instruct-v0.3,INT4-MIXED,32,43.9,35.8,22.78 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,44,38.2,22.73 -llama-3-8b-instruct,INT4-MIXED,32,44,38.2,22.73 -llama-3.1-8b-instruct,INT4-MIXED,32,44,38.2,22.73 -phi-4-mini-reasoning,INT8-CW,32,44.3,39,22.57 -phi-4-mini-instruct,INT8-CW,32,44.4,39.1,22.52 -qwen2.5-7b-instruct,INT4-MIXED,32,44.5,36.3,22.47 -mistral-7b-instruct-v0.3,INT4-MIXED,32,44.6,36.3,22.42 -qwen2-7b-instruct,INT4-MIXED,32,44.6,37.4,22.42 -qwen2.5-7b-instruct,INT4-MIXED,32,44.7,37.4,22.37 -deepseek-r1-distill-qwen-7b,INT4-MIXED,32,44.8,37.3,22.32 -mistral-7b-instruct-v0.1,INT4-MIXED,32,44.8,36.2,22.32 -qwen3-4b,INT8-CW,32,44.8,39,22.32 -qwen3-embedding-0.6b,INT8-CW,1024,44.8,-1,22.32 -mistral-7b-instruct-v0.2,INT4-MIXED,32,44.9,36.3,22.27 -bloomz-7b1,INT4-MIXED,32,45.2,38.9,22.12 -phi-4-mini-instruct,INT4-MIXED,32,45.2,24.6,22.12 -qwen3-8b,INT4-MIXED,32,45.2,39.3,22.12 -qwen2-7b-instruct,INT4-MIXED,32,45.5,37.9,21.98 -afm-4.5b,INT8-CW,32,45.7,40.9,21.88 -deepseek-r1-distill-qwen-7b,INT4-MIXED,32,45.8,37.8,21.83 -qwen2.5-7b-instruct-1m,INT4-MIXED,32,45.8,37.8,21.83 -qwen3-reranker-0.6b-seq-cls,FP16,836,45.8,-1,21.83 -qwen2.5-7b-instruct,INT4-MIXED,32,45.9,37.8,21.79 -minicpm4-8b,INT4-MIXED,32,46.4,38.8,21.55 -falcon-7b-instruct,INT4-MIXED,32,47.1,37.9,21.23 -qwen3-embedding-0.6b,FP16,1024,47.2,-1,21.19 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,47.3,39.3,21.14 -llama-3-8b-instruct,INT4-MIXED,32,47.3,39.3,21.14 -llama-3.1-8b-instruct,INT4-MIXED,32,47.5,39.2,21.05 -qwen3-reranker-0.6b-seq-cls,INT4-MIXED,836,47.6,-1,21.01 -llama-3-8b-instruct,INT4-MIXED,32,47.7,39.2,20.96 -lcm-dreamshaper-v7,INT8-CW,1024,47.9,47.2,20.88 -llama-3-8b-instruct,INT4-MIXED,32,48.2,39.8,20.75 -qwen3-8b,INT4-MIXED,32,48.2,40.5,20.75 -qwen3-reranker-0.6b-seq-cls,INT8-CW,836,48.2,-1,20.75 -lcm-dreamshaper-v7,INT8-CW,32,48.3,47.6,20.70 -lcm-dreamshaper-v7,INT8-CW,1024,48.5,47.7,20.62 -bloomz-7b1,INT4-MIXED,32,48.9,40.3,20.45 -lcm-dreamshaper-v7,INT8-CW,32,48.9,47.1,20.45 -gemma-2b-it,FP16,32,49.6,46.3,20.16 -minicpm4-8b,INT4-MIXED,32,49.6,40,20.16 -qwen3-embedding-0.6b,INT4-MIXED,1024,49.6,-1,20.16 -lcm-dreamshaper-v7,FP16,1024,49.7,49,20.12 -lcm-dreamshaper-v7,FP16,32,49.7,48.9,20.12 -qwen3-8b,INT4-MIXED,32,49.8,40.9,20.08 -baichuan2-7b-chat,INT4-MIXED,32,50.9,43.4,19.65 -glm-4-9b-chat-hf,INT4-MIXED,32,51.2,44.5,19.53 -minicpm4-8b,INT4-MIXED,32,51.3,40.6,19.49 -gemma-3-1b-it,INT4-MIXED,1024,51.7,8.4,19.34 -gemma-7b-it,INT4-MIXED,32,51.8,44.4,19.31 -qwen3_8b_eagle3,INT4-MIXED,32,51.9,33.8,19.27 -llama-3.2-1b-instruct,INT4-MIXED,1024,52.1,8.6,19.19 -gemma-3-4b-it,INT4-MIXED,32,52.7,25.3,18.98 -gemma-2-2b,FP16,33,53.3,50.2,18.76 -gemma-3-1b-it,INT4-MIXED,1024,53.8,8.6,18.59 -glm-4-9b-chat-hf,INT4-MIXED,32,54.1,45.8,18.48 -phi-2,FP16,32,54.1,50.8,18.48 -stable-zephyr-3b-dpo,FP16,32,54.3,51,18.42 -stablelm-3b-4e1t,FP16,32,54.5,51.1,18.35 -gemma-7b-it,INT4-MIXED,32,55,46.2,18.18 -gemma-3-4b-it,INT4-MIXED,32,55.1,25.6,18.15 -glm-4-9b-chat-hf,INT4-MIXED,32,55.3,46.4,18.08 -ltx-video,FP16,11,55.4,54.8,18.05 -gemma-3-4b-it,INT4-MIXED,32,55.6,24.8,17.99 -llama-3.2-1b-instruct,INT8-CW,1024,56.3,13,17.76 -nanollava,INT8-CW,760,56.5,7.4,17.70 -nanollava,INT4-MIXED,760,57.3,6.5,17.45 -gemma-2-9b-it,INT4-MIXED,32,58,49.7,17.24 -gemma-3-4b-it,INT8-CW,32,58.5,39.5,17.09 -glm-edge-4b-chat,INT4-MIXED,32,58.5,24.5,17.09 -llama-3.2-3b-instruct,FP4-NORMALIZED,32,59.4,56.3,16.84 -chatglm3-6b,INT8-CW,32,60.3,55.3,16.58 -gemma-3-1b-it,INT4-MIXED,1024,60.3,9.1,16.58 -nanollava,FP16,760,60.3,11.5,16.58 -deepseek-r1-distill-llama-8b,INT4-MIXED,32,60.5,47.1,16.53 -llama-3.1-8b-instruct,INT4-MIXED,32,60.8,47.1,16.45 -gemma-3-1b-it,INT8-CW,1024,61.1,11.8,16.37 -qwen2.5-coder-3b-instruct,FP16,32,61.1,57.3,16.37 -glm-edge-4b-chat,INT8-CW,32,61.2,43.3,16.34 -gemma-2-9b-it,INT4-MIXED,32,61.6,51.1,16.23 -gemma-2-9b-it,INT4-MIXED,32,62.5,51.7,16.00 -llama-3.2-3b-instruct,FP16,32,62.5,60,16.00 -whisper-small,FP16,prompt0,64.1,8.1,15.60 -whisper-small,INT4-MIXED,prompt0,66.1,6,15.13 -qwen3.5-0.8b,FP16,85,66.2,15.8,15.11 -whisper-small,INT4-MIXED,prompt0,66.8,6.2,14.97 -llama-2-7b-chat-hf,INT8-CW,32,67.1,62.1,14.90 -biomistral-7b-slerp,INT8-CW,7,67.5,65.5,14.81 -codellama-7b,INT8-CW,32,67.5,61.7,14.81 -qwen3.5-0.8b,FP4-NORMALIZED,85,68.1,13.4,14.68 -lcm-sdxl,INT8-HYBRID,32,68.7,68,14.56 -llama-2-13b-chat-hf,INT4-MIXED,32,69.2,61.5,14.45 -lcm-sdxl,INT8-HYBRID,1024,69.4,68.2,14.41 -lcm-sdxl,INT8-CW,1024,69.8,69.2,14.33 -lcm-sdxl,INT8-CW,32,69.9,69.3,14.31 -whisper-small,INT8-CW,prompt0,70,6.2,14.29 -falcon-7b-instruct,INT8-CW,32,70.1,64.3,14.27 -minicpm-1b-sft,INT4-MIXED,1014,70.6,10.5,14.16 -lcm-sdxl,INT8-CW,1024,70.9,70.5,14.10 -lcm-sdxl,INT8-CW,32,71.1,70.5,14.06 -phi-4-mini-reasoning,FP4-NORMALIZED,32,71.1,66.2,14.06 -phi-4-mini-instruct,FP4-NORMALIZED,32,71.2,66.4,14.04 -stable-diffusion-v1-5,INT8-HYBRID,1024,71.2,70.4,14.04 -stable-diffusion-v1-5,INT8-HYBRID,32,71.2,70.2,14.04 -whisper-small,INT4-MIXED,prompt0,71.2,6.4,14.04 -baichuan2-7b-chat,INT8-CW,32,71.6,65.1,13.97 -dolly-v2-12b,INT4-MIXED,32,71.7,59.7,13.95 -mistral-7b-instruct-v0.1,INT8-CW,32,72,65.9,13.89 -mistral-7b-instruct-v0.2,INT8-CW,32,72,65.9,13.89 -mistral-7b-instruct-v0.3,INT8-CW,32,72,65.7,13.89 -deepseek-r1-distill-qwen-7b,INT8-CW,32,72.5,66.5,13.79 -qwen2.5-7b-instruct,INT8-CW,32,72.5,66.8,13.79 -qwen2.5-7b-instruct-1m,INT8-CW,32,72.6,66.4,13.77 -qwen2-7b-instruct,INT8-CW,32,72.7,66.4,13.76 -phi-3-mini-4k-instruct,FP16,32,72.8,69.6,13.74 -bloomz-7b1,INT8-CW,32,72.9,68,13.72 -gemma-3-1b-it,FP4-NORMALIZED,1024,72.9,18,13.72 -phi-3-mini-128k-instruct,FP16,32,72.9,69.3,13.72 -phi-3.5-mini-instruct,FP16,32,73.1,69.3,13.68 -gemma-3-1b-it,FP16,1024,73.6,20.7,13.59 -qwen3.5-0.8b,INT4-MIXED,85,74,6.9,13.51 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,74.7,10.1,13.39 -sd-turbo,INT8-HYBRID,1024,74.7,74.3,13.39 -sd-turbo,INT8-HYBRID,32,74.7,74.1,13.39 -stable-diffusion-v2-1,INT8-HYBRID,1024,74.8,74.3,13.37 -stable-diffusion-v2-1,INT8-HYBRID,32,75,74.2,13.33 -qwen2.5-1.5b-instruct,INT8-CW,1024,75.5,16.3,13.25 -llama-3.1-8b-instruct,INT8-CW,32,75.6,69.4,13.23 -deepseek-r1-distill-llama-8b,INT8-CW,32,75.7,69.3,13.21 -llama-3-8b-instruct,INT8-CW,32,75.7,69.3,13.21 -qwen2.5-coder-1.5b-instruct,INT8-CW,1024,76,16.3,13.16 -llama-3.2-1b-instruct,FP16,1024,76.2,23.6,13.12 -phi-4-mini-instruct,FP16,32,76.4,71.8,13.09 -phi-4-mini-reasoning,FP16,32,76.4,72,13.09 -qwen3.5-0.8b,INT4-MIXED,85,76.6,7.1,13.05 -qwen3-reranker-0.6b,INT4-MIXED,836,77.1,-1,12.97 -qwen3-8b,INT8-CW,32,77.6,70.1,12.89 -minicpm-1b-sft,INT4-MIXED,1014,78.2,11.3,12.79 -phi-4,INT4-MIXED,32,78.2,68.3,12.79 -phi-4-reasoning,INT4-MIXED,32,78.2,68.3,12.79 -gemma-3-4b-it,FP4-NORMALIZED,32,78.5,67.5,12.74 -qwen3-reranker-0.6b,FP16,836,78.7,-1,12.71 -qwen3-4b,FP16,32,78.8,74.4,12.69 -qwen3.5-0.8b,INT8-CW,85,78.8,9.2,12.69 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,79,10.5,12.66 -deepseek-r1-distill-qwen-1.5b,INT8-CW,1024,79.1,16.2,12.64 -qwen3.5-0.8b,INT4-MIXED,85,79.8,7.3,12.53 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,80,10.4,12.50 -qwen1.5-14b-chat,INT4-MIXED,32,80.1,69.5,12.48 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,80.3,10.3,12.45 -qwen3-reranker-0.6b,INT4-MIXED,836,80.7,-1,12.39 -qwen3_8b_eagle3,INT8-CW,32,80.8,55.7,12.38 -minicpm4-8b,INT8-CW,32,81.5,75.2,12.27 -phi-4-reasoning,INT4-MIXED,32,82.5,70.4,12.12 -stable-diffusion-v1-5,INT8-CW,1024,83,82.5,12.05 -minicpm-1b-sft,INT8-CW,1014,83.2,16.7,12.02 -afm-4.5b,FP16,32,83.3,79.8,12.00 -qwen3-reranker-0.6b,INT8-CW,836,83.3,-1,12.00 -stable-diffusion-v1-5,INT8-CW,32,83.3,82.5,12.00 -stable-diffusion-v1-5,INT8-CW,32,83.8,82.7,11.93 -qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,84.1,10.4,11.89 -gemma-3-4b-it,FP16,32,84.2,73.4,11.88 -qwen2.5-1.5b-instruct,INT4-MIXED,1024,84.5,11.2,11.83 -sd-turbo,INT8-CW,1024,84.5,84.2,11.83 -stable-diffusion-v1-5,FP16,1024,84.5,84.6,11.83 -sd-turbo,INT8-CW,32,84.6,84,11.82 -stable-diffusion-v2-1,INT8-CW,32,84.6,84.1,11.82 -phi-4,INT4-MIXED,32,84.8,71.4,11.79 -glm-edge-4b-chat,FP16,32,85,78.3,11.76 -stable-diffusion-v1-5,FP16,32,85,84.3,11.76 -stable-diffusion-v1-5,INT8-CW,1024,85.1,83,11.75 -stable-diffusion-v2-1,INT8-CW,1024,85.2,84.1,11.74 -stable-diffusion-v2-1,INT8-CW,32,85.3,84.5,11.72 -stable-diffusion-v2-1,INT8-CW,1024,85.4,84.6,11.71 -deepseek-r1-distill-qwen-14b,INT4-MIXED,32,85.5,72.4,11.70 -sd-turbo,INT8-CW,1024,85.5,85.2,11.70 -sd-turbo,INT8-CW,32,85.5,84.9,11.70 -sd-turbo,FP16,32,86,85.7,11.63 -sd-turbo,FP16,1024,86.2,89.7,11.60 -glm-4-9b-chat-hf,INT8-CW,32,88,80.7,11.36 -qwen3-reranker-0.6b,INT4-MIXED,836,88,-1,11.36 -lcm-sdxl,FP16,32,88.7,88.3,11.27 -lcm-sdxl,FP16,1024,88.8,88.4,11.26 -qwen2.5-1.5b-instruct,FP4-NORMALIZED,1024,89,27.9,11.24 -deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,89.1,13.1,11.22 -deepseek-r1-distill-qwen-1.5b,FP4-NORMALIZED,1024,89.3,27.9,11.20 -gemma-7b-it,INT8-CW,32,89.4,81,11.19 -gemma-3-12b-it,INT4-MIXED,32,89.5,63.4,11.17 -starcoder2-15b,INT4-MIXED,32,89.7,77.6,11.15 -minicpm-1b-sft,INT4-MIXED,1014,89.8,11.3,11.14 -gemma-3-12b-it,INT4-MIXED,32,91.6,66,10.92 -qwen2.5-1.5b-instruct,FP16,1024,91.6,30.1,10.92 -deepseek-r1-distill-qwen-1.5b,FP16,1024,91.9,30.1,10.88 -qwen2.5-coder-1.5b-instruct,FP16,1024,92,30.1,10.87 -nanollava,INT8-CW,1752,92.2,8.5,10.85 -llama-2-13b-chat-hf,INT4-MIXED,32,92.8,76.4,10.78 -gemma-2b-it,INT4-MIXED,1024,93.8,15.7,10.66 -minicpm-1b-sft,FP16,1014,95,28.9,10.53 -gemma-2-9b-it,INT8-CW,32,96.7,87.9,10.34 -phi-4-reasoning,INT4-MIXED,32,97.6,74.3,10.25 -nanollava,INT4-MIXED,1752,99.9,7.4,10.01 \ No newline at end of file +minicpm4-0.5b,INT4-MIXED,32,15.5,4,250 +minicpm4-0.5b,INT4-MIXED,32,16.1,4.1,243.902439 +minicpm4-0.5b,INT4-MIXED,1024,22,4.1,243.902439 +minicpm4-0.5b,INT4-MIXED,32,15.4,4.1,243.902439 +minicpm4-0.5b,INT4-MIXED,1024,27.5,4.2,238.0952381 +gemma-3-270m,INT4-MIXED,1024,19.3,4.4,227.2727273 +gemma-3-270m,INT4-MIXED,32,16.8,4.4,227.2727273 +gemma-3-270m,INT8-CW,1024,20.2,4.4,227.2727273 +minicpm4-0.5b,INT4-MIXED,1024,22.2,4.4,227.2727273 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,15.8,4.5,222.2222222 +whisper-large-v3-turbo,INT4-MIXED,1024,170.9,4.5,222.2222222 +gemma-3-270m,INT8-CW,32,16.7,4.6,217.3913043 +lfm2.5-350m,INT4-MIXED,32,18.1,4.6,217.3913043 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,16.6,4.6,217.3913043 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,24.8,4.6,217.3913043 +distil-large-v2,INT4-MIXED,32,110.1,4.7,212.7659574 +distil-large-v2,INT4-MIXED,1024,163.1,4.7,212.7659574 +nuextract-tiny,INT4-MIXED,32,17,4.7,212.7659574 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,30.5,4.7,212.7659574 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,32,15.9,4.7,212.7659574 +whisper-large-v3-turbo,INT4-MIXED,32,115.9,4.7,212.7659574 +whisper-large-v3-turbo,INT8-CW,1024,166.8,4.7,212.7659574 +distil-large-v2,INT8-CW,32,103.6,4.8,208.3333333 +distil-large-v2,INT8-CW,1024,156.2,4.8,208.3333333 +nuextract-tiny,INT4-MIXED,1024,26.7,4.8,208.3333333 +qwen2.5-coder-0.5b-instruct,INT4-MIXED,1024,25.3,4.8,208.3333333 +whisper-large-v3-turbo,INT8-CW,32,111.8,4.8,208.3333333 +lfm2.5-350m,INT4-MIXED,1024,35.2,4.9,204.0816327 +lfm2.5-350m,INT8-CW,32,19.5,5.6,178.5714286 +minicpm4-0.5b,INT8-CW,32,17.2,5.6,178.5714286 +lfm2.5-350m,INT8-CW,1024,32.8,5.8,172.4137931 +nuextract-tiny,INT8-CW,32,17.8,5.8,172.4137931 +minicpm4-0.5b,INT8-CW,1024,24.5,5.9,169.4915254 +whisper-small,INT4-MIXED,32,66.1,5.9,169.4915254 +whisper-small,INT4-MIXED,1024,115.3,6,166.6666667 +qwen2.5-coder-0.5b-instruct,INT8-CW,32,18,6.1,163.9344262 +whisper-small,INT4-MIXED,1024,117.3,6.1,163.9344262 +nuextract-tiny,INT8-CW,1024,24.4,6.2,161.2903226 +whisper-small,INT4-MIXED,1024,121.5,6.2,161.2903226 +qwen2.5-coder-0.5b-instruct,INT8-CW,1024,27.3,6.3,158.7301587 +whisper-small,INT8-CW,1024,124.3,6.3,158.7301587 +whisper-small,INT4-MIXED,32,67,6.4,156.25 +whisper-small,INT8-CW,32,69,6.4,156.25 +tiny-llama-1.1b-chat,INT4-MIXED,32,14.3,6.5,153.8461538 +whisper-small,INT4-MIXED,32,70.6,6.5,153.8461538 +nanollava,INT4-MIXED,760,57.5,6.6,151.5151515 +whisper-large-v3-turbo,FP16,32,119.3,6.6,151.5151515 +whisper-large-v3-turbo,FP16,1024,175.8,6.6,151.5151515 +gemma-3-270m,FP16,1024,17.4,6.7,149.2537313 +gemma-3-270m,FP16,32,14.9,6.7,149.2537313 +minicpm5-1b,INT4-MIXED,32,17.3,6.7,149.2537313 +tiny-llama-1.1b-chat,INT4-MIXED,1024,41.2,6.7,149.2537313 +distil-large-v2,FP16,1024,165.9,6.8,147.0588235 +tiny-llama-1.1b-chat,INT4-MIXED,32,14.9,6.8,147.0588235 +distil-large-v2,FP16,32,113.2,6.9,144.9275362 +minicpm5-1b,INT4-MIXED,1024,44.1,7,142.8571429 +tiny-llama-1.1b-chat,INT4-MIXED,1024,55.5,7,142.8571429 +qwen3.5-0.8b,INT4-MIXED,85,45.7,7.1,140.8450704 +qwen3.5-0.8b,INT4-MIXED,85,45.9,7.2,138.8888889 +nanollava,INT4-MIXED,1752,100.4,7.4,135.1351351 +nanollava,INT8-CW,760,56.8,7.4,135.1351351 +qwen3.5-0.8b,INT4-MIXED,85,52,7.4,135.1351351 +qwen3.5-0.8b,INT4-MIXED,1024,89.4,7.5,133.3333333 +qwen3.5-0.8b,INT4-MIXED,1024,89.7,7.5,133.3333333 +lfm2.5-1.2b-instruct,INT4-MIXED,32,18.8,8,125 +llama-3.2-1b-instruct,INT4-MIXED,32,13.2,8,125 +qwen3.5-0.8b,INT4-MIXED,1024,99.9,8,125 +llama-3.2-1b-instruct,INT4-MIXED,32,13.5,8.1,123.4567901 +whisper-small,FP16,32,64.5,8.1,123.4567901 +whisper-small,FP16,1024,116.5,8.1,123.4567901 +lfm2.5-1.2b-instruct,INT4-MIXED,32,18.3,8.2,121.9512195 +lfm2-1.2b,INT4-MIXED,1024,65,8.3,120.4819277 +lfm2-1.2b,INT4-MIXED,32,18.3,8.3,120.4819277 +lfm2.5-1.2b-instruct,INT4-MIXED,32,19,8.3,120.4819277 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,66,8.3,120.4819277 +llama-3.2-1b-instruct,INT4-MIXED,1024,41.6,8.3,120.4819277 +gemma-3-1b-it,INT4-MIXED,1024,47.1,8.4,119.047619 +lfm2-1.2b,INT4-MIXED,32,19.2,8.4,119.047619 +llama-3.2-1b-instruct,INT4-MIXED,1024,49.3,8.4,119.047619 +lfm2-1.2b,INT4-MIXED,1024,66.2,8.5,117.6470588 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,66.8,8.5,117.6470588 +nanollava,INT8-CW,1752,92.6,8.5,117.6470588 +gemma-3-1b-it,INT4-MIXED,1024,47.9,8.6,116.2790698 +gemma-3-1b-it,INT4-MIXED,32,21.8,8.6,116.2790698 +lfm2-1.2b,INT4-MIXED,32,18.8,8.6,116.2790698 +lfm2.5-350m,FP16,32,17.3,8.6,116.2790698 +lfm2-1.2b,INT4-MIXED,1024,74.9,8.7,114.9425287 +minicpm4-0.5b,FP4-NORMALIZED,32,14,8.7,114.9425287 +gemma-3-1b-it,INT4-MIXED,32,22.6,8.8,113.6363636 +gemma-3-1b-it,INT4-MIXED,32,21.9,8.8,113.6363636 +lfm2.5-350m,FP16,1024,35.7,8.8,113.6363636 +minicpm4-0.5b,FP4-NORMALIZED,1024,29.5,8.8,113.6363636 +lfm2.5-1.2b-instruct,INT4-MIXED,1024,75.6,8.9,112.3595506 +exaone-4.0-1.2b,INT4-MIXED,32,24.1,9,111.1111111 +gemma-3-1b-it,INT4-MIXED,1024,53.9,9,111.1111111 +exaone-4.0-1.2b,INT4-MIXED,32,24.6,9.1,109.8901099 +exaone-4.0-1.2b,INT4-MIXED,32,25.4,9.2,108.6956522 +exaone-4.0-1.2b,INT4-MIXED,1024,70.9,9.2,108.6956522 +qwen3.5-0.8b,INT8-CW,85,51.6,9.3,107.5268817 +exaone-4.0-1.2b,INT4-MIXED,1024,72.5,9.4,106.3829787 +exaone-4.0-1.2b,INT4-MIXED,1024,79.4,9.5,105.2631579 +minicpm4-0.5b,FP16,32,14.7,9.5,105.2631579 +minicpm5-1b,INT8-CW,32,18.4,9.5,105.2631579 +qwen3.5-0.8b,INT8-CW,1024,94.1,9.6,104.1666667 +minicpm4-0.5b,FP16,1024,30.3,9.7,103.0927835 +minicpm-1b-sft,INT4-MIXED,31,34,9.8,102.0408163 +minicpm-1b-sft,INT4-MIXED,31,34.1,9.9,101.010101 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,20,9.9,101.010101 +minicpm-1b-sft,INT4-MIXED,1014,65.9,10,100 +minicpm5-1b,INT8-CW,1024,40.7,10,100 +nuextract-tiny,FP16,32,16.5,10,100 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,66.9,10,100 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,19.8,10.1,99.00990099 +minicpm-1b-sft,INT4-MIXED,31,38.4,10.1,99.00990099 +qwen2.5-1.5b-instruct,INT4-MIXED,32,19.8,10.1,99.00990099 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,19.8,10.1,99.00990099 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,32,20.9,10.2,98.03921569 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,78.1,10.3,97.08737864 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,78.3,10.3,97.08737864 +minicpm-1b-sft,INT4-MIXED,1014,67,10.4,96.15384615 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,83,10.4,96.15384615 +qwen2.5-coder-1.5b-instruct,INT4-MIXED,1024,82.4,10.4,96.15384615 +qwen2.5-coder-0.5b-instruct,FP16,32,16,10.5,95.23809524 +qwen2.5-coder-0.5b-instruct,FP16,1024,34.3,10.7,93.45794393 +minicpm-1b-sft,INT4-MIXED,1014,78.2,10.8,92.59259259 +nuextract-tiny,FP16,1024,32.1,10.8,92.59259259 +qwen2.5-1.5b-instruct,INT4-MIXED,32,21.8,10.9,91.74311927 +nanollava,FP16,760,59,11.5,86.95652174 +qwen2.5-1.5b-instruct,INT4-MIXED,1024,83.5,11.5,86.95652174 +tiny-llama-1.1b-chat,INT8-CW,32,22.7,11.5,86.95652174 +gemma-3-1b-it,INT8-CW,32,24.8,11.6,86.20689655 +gemma-3-1b-it,INT8-CW,1024,59,11.7,85.47008547 +tiny-llama-1.1b-chat,INT8-CW,1024,51.4,11.8,84.74576271 +lfm2-8b-a1b,INT4-MIXED,32,77.9,12,83.33333333 +llama-3.2-1b-instruct,INT8-CW,32,14.8,12.7,78.74015748 +nanollava,FP16,1752,102.9,12.7,78.74015748 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,32,22.3,12.9,77.51937984 +deepseek-r1-distill-qwen-1.5b,INT4-MIXED,1024,87.9,13,76.92307692 +lfm2-8b-a1b,INT4-MIXED,1024,184.1,13,76.92307692 +llama-3.2-1b-instruct,INT8-CW,1024,54.6,13,76.92307692 +lfm2.5-1.2b-instruct,INT8-CW,32,21.5,13.3,75.18796992 +lfm2-1.2b,INT8-CW,32,22.1,13.5,74.07407407 +qwen3.5-0.8b,FP4-NORMALIZED,85,37.9,13.5,74.07407407 +lfm2.5-1.2b-instruct,INT8-CW,1024,83.9,13.6,73.52941176 +lfm2-1.2b,INT8-CW,1024,84.7,13.8,72.46376812 +qwen3.5-0.8b,FP4-NORMALIZED,1024,102.7,13.8,72.46376812 +exaone-4.0-1.2b,INT8-CW,32,27.5,13.9,71.94244604 +exaone-4.0-1.2b,INT8-CW,1024,81.4,14.5,68.96551724 +phi-2,INT4-MIXED,32,29.6,15.5,64.51612903 +minicpm-1b-sft,INT8-CW,31,38.9,15.6,64.1025641 +qwen2.5-1.5b-instruct,INT8-CW,32,22.6,15.7,63.69426752 +qwen2.5-coder-1.5b-instruct,INT8-CW,32,22.8,15.7,63.69426752 +stable-zephyr-3b-dpo,INT4-MIXED,32,30.7,15.7,63.69426752 +deepseek-r1-distill-qwen-1.5b,INT8-CW,32,23.1,15.8,63.29113924 +qwen3.5-0.8b,FP16,85,38.1,15.9,62.89308176 +qwen3.5-0.8b,FP16,1024,104,16,62.5 +deepseek-r1-distill-qwen-1.5b,INT8-CW,1024,74.4,16.3,61.34969325 +qwen2.5-1.5b-instruct,INT8-CW,1024,74.1,16.3,61.34969325 +qwen2.5-coder-1.5b-instruct,INT8-CW,1024,73.3,16.3,61.34969325 +gemma-4-e2b-it,INT4-MIXED,274,171.5,16.5,60.60606061 +minicpm-1b-sft,INT8-CW,1014,78.8,16.8,59.52380952 +gemma-4-e2b-it,INT4-MIXED,1024,295.5,16.9,59.17159763 +gemma-4-e2b-it,INT4-MIXED,274,186.9,17,58.82352941 +phi-2,INT4-MIXED,32,38.5,17,58.82352941 +whisper-large-v3,INT4-MIXED,1024,261.4,17.1,58.47953216 +phi-2,INT4-MIXED,1024,161.6,17.2,58.13953488 +whisper-large-v3,INT4-MIXED,32,183.6,17.3,57.80346821 +gemma-4-e2b-it,INT4-MIXED,1024,319.7,17.4,57.47126437 +minicpm5-1b,FP16,32,19.5,17.4,57.47126437 +smollm3-3b,INT4-MIXED,32,25.7,17.5,57.14285714 +stable-zephyr-3b-dpo,INT4-MIXED,32,41,17.6,56.81818182 +stable-zephyr-3b-dpo,INT4-MIXED,1024,153.2,17.6,56.81818182 +gemma-3-1b-it,FP4-NORMALIZED,32,21.4,17.7,56.49717514 +gemma-3-1b-it,FP4-NORMALIZED,1024,69.2,17.8,56.17977528 +lfm2-8b-a1b,INT8-CW,32,76.7,17.8,56.17977528 +minicpm5-1b,FP16,1024,52.3,17.8,56.17977528 +smollm3-3b,INT4-MIXED,1024,122.8,18,55.55555556 +smollm3-3b,INT4-MIXED,32,31.6,18,55.55555556 +llama-3.2-3b-instruct,INT4-MIXED,32,26.4,18.2,54.94505495 +smollm3-3b,INT4-MIXED,32,32.4,18.3,54.64480874 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,32.8,18.4,54.34782609 +smollm3-3b,INT4-MIXED,1024,124.8,18.6,53.76344086 +llama-3.2-3b-instruct,INT4-MIXED,32,27.1,18.7,53.47593583 +lfm2-8b-a1b,INT8-CW,1024,251.6,18.8,53.19148936 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,32.7,18.8,53.19148936 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,120.5,18.8,53.19148936 +smollm3-3b,INT4-MIXED,1024,147.4,18.8,53.19148936 +llama-3.2-3b-instruct,INT4-MIXED,32,27.8,18.9,52.91005291 +phi-2,INT4-MIXED,1024,209.8,18.9,52.91005291 +stable-zephyr-3b-dpo,INT4-MIXED,1024,175.7,18.9,52.91005291 +lfm2-24b-a2b,INT4-MIXED,32,140.3,19.1,52.35602094 +llama-3.2-3b-instruct,INT4-MIXED,1024,119.7,19.1,52.35602094 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,118.9,19.1,52.35602094 +qwen2.5-coder-3b-instruct,INT4-MIXED,32,36.9,19.4,51.54639175 +llama-3.2-3b-instruct,INT4-MIXED,1024,141.1,19.6,51.02040816 +lfm2-24b-a2b,INT4-MIXED,1024,420,19.7,50.76142132 +tiny-llama-1.1b-chat,FP16,32,21.5,19.7,50.76142132 +qwen2.5-coder-3b-instruct,INT4-MIXED,1024,212.1,19.8,50.50505051 +llama-3.2-3b-instruct,INT4-MIXED,1024,142.8,19.9,50.25125628 +tiny-llama-1.1b-chat,FP16,1024,63.3,20.1,49.75124378 +gemma-3-1b-it,FP16,32,24.2,20.3,49.26108374 +qwen3-30b-a3b,INT4-MIXED,32,154.1,20.4,49.01960784 +qwen3-30b-a3b,INT4-MIXED,32,154.5,20.4,49.01960784 +whisper-large-v3,INT8-CW,32,180.9,20.4,49.01960784 +whisper-large-v3,INT8-CW,1024,237.9,20.6,48.54368932 +gemma-3-1b-it,FP16,1024,71.9,20.7,48.30917874 +gpt-oss-20b,INT4-MIXED,32,121.9,22.2,45.04504505 +qwen3-30b-a3b,INT4-MIXED,1024,461.5,22.4,44.64285714 +qwen3-30b-a3b,INT4-MIXED,1024,467.1,22.4,44.64285714 +qwen3.6-35b-a3b,INT4-MIXED,42,160.2,22.5,44.44444444 +qwen3-4b,INT4-MIXED,32,36.5,22.8,43.85964912 +phi-4-mini-instruct,INT4-MIXED,32,40.4,23.1,43.29004329 +phi-4-mini-reasoning,INT4-MIXED,32,40.8,23.1,43.29004329 +qwen3.6-35b-a3b,INT4-MIXED,42,169.1,23.1,43.29004329 +gpt-oss-20b,INT4-MIXED,1024,354.3,23.2,43.10344828 +llama-3.2-1b-instruct,FP16,32,25.2,23.3,42.91845494 +lfm2-1.2b,FP16,32,28.4,23.4,42.73504274 +lfm2.5-1.2b-instruct,FP16,32,29,23.4,42.73504274 +qwen3-4b,INT4-MIXED,32,36.9,23.4,42.73504274 +qwen3.6-35b-a3b,INT4-MIXED,1039,700.2,23.4,42.73504274 +llama-3.2-1b-instruct,FP16,1024,74.7,23.6,42.37288136 +lfm2-1.2b,FP16,1024,105.1,23.7,42.19409283 +lfm2.5-1.2b-instruct,FP16,1024,104.8,23.7,42.19409283 +phi-4-mini-instruct,INT4-MIXED,32,40.9,23.7,42.19409283 +phi-4-mini-reasoning,INT4-MIXED,32,41.1,23.7,42.19409283 +phi-4-mini-reasoning,INT4-MIXED,32,42.4,23.9,41.84100418 +qwen3.6-35b-a3b,INT4-MIXED,1039,767.7,23.9,41.84100418 +phi-4-mini-instruct,INT4-MIXED,1024,185.7,24,41.66666667 +qwen3.5-35b-a3b,INT4-MIXED,83,340.1,24,41.66666667 +phi-4-mini-reasoning,INT4-MIXED,1024,186,24.1,41.49377593 +afm-4.5b,INT4-MIXED,32,31.3,24.2,41.32231405 +qwen3-4b,INT4-MIXED,1024,170,24.2,41.32231405 +phi-3.5-vision-instruct,INT4-MIXED,802,285.8,24.6,40.6504065 +gemma-3-4b-it,INT4-MIXED,32,46.5,24.7,40.48582996 +phi-4-mini-instruct,INT4-MIXED,1024,190.7,24.7,40.48582996 +phi-4-mini-reasoning,INT4-MIXED,1024,190.9,24.7,40.48582996 +phi-4-mini-instruct,INT4-MIXED,32,44.7,24.8,40.32258065 +qwen3-4b,INT4-MIXED,1024,175.9,24.8,40.32258065 +phi-4-mini-reasoning,INT4-MIXED,1024,213.4,24.9,40.16064257 +exaone-4.0-1.2b,FP16,32,30.1,25,40 +gemma-4-e2b-it,INT8-CW,274,197.4,25,40 +qwen3.5-35b-a3b,INT4-MIXED,1024,862.8,25,40 +afm-4.5b,INT4-MIXED,1024,192.8,25.2,39.68253968 +gemma-3-4b-it,INT4-MIXED,32,47.5,25.3,39.5256917 +trinity-mini,INT4-MIXED,32,165.7,25.4,39.37007874 +exaone-4.0-1.2b,FP16,1024,104,25.5,39.21568627 +gemma-3-4b-it,INT4-MIXED,32,51.4,25.5,39.21568627 +phi-3.5-vision-instruct,INT4-MIXED,1032,337.2,25.5,39.21568627 +phi-4-mini-instruct,INT4-MIXED,1024,250.1,25.7,38.91050584 +gemma-4-e2b-it,INT8-CW,1024,319.4,25.8,38.75968992 +gemma-3-4b-it,INT4-MIXED,1024,204.5,26.1,38.31417625 +gemma-3-4b-it,INT4-MIXED,1024,215.2,26.7,37.45318352 +trinity-mini,INT4-MIXED,1024,432.1,26.7,37.45318352 +gpt-oss-20b,INT4-MIXED,32,125.7,26.8,37.31343284 +gemma-3-4b-it,INT4-MIXED,1024,231.1,27,37.03703704 +phi-2,INT8-CW,32,40.1,27.1,36.900369 +qwen2.5-1.5b-instruct,FP4-NORMALIZED,32,32.1,27.5,36.36363636 +deepseek-r1-distill-qwen-1.5b,FP4-NORMALIZED,32,30.3,27.6,36.23188406 +gpt-oss-20b,INT4-MIXED,1024,375.9,27.7,36.10108303 +deepseek-r1-distill-qwen-1.5b,FP4-NORMALIZED,1024,86.5,27.9,35.84229391 +gemma-4-26b-a4b-it,INT4-MIXED,278,611.1,27.9,35.84229391 +qwen2.5-1.5b-instruct,FP4-NORMALIZED,1024,86.7,27.9,35.84229391 +minicpm-1b-sft,FP16,31,36.2,28,35.71428571 +phi-4-multimodal-instruct,INT4-MIXED,578,274.6,28.1,35.58718861 +minicpm3-4b,INT4-MIXED,32,108.6,28.3,35.33568905 +phi-4-multimodal-instruct,INT4-MIXED,786,314.6,28.5,35.0877193 +gemma-4-26b-a4b-it,INT4-MIXED,1024,911.2,29,34.48275862 +minicpm-1b-sft,FP16,1014,89.2,29,34.48275862 +phi-2,INT8-CW,1024,173.7,29,34.48275862 +minicpm3-4b,INT4-MIXED,32,113.1,29.1,34.36426117 +phi-4-multimodal-instruct,INT4-MIXED,1362,620.3,29.4,34.01360544 +minicpm3-4b,INT4-MIXED,32,112.8,29.7,33.67003367 +phi-4-multimodal-instruct,INT4-MIXED,1570,675.3,29.7,33.67003367 +qwen2.5-1.5b-instruct,FP16,32,31.9,29.7,33.67003367 +qwen2.5-coder-1.5b-instruct,FP16,32,32,29.7,33.67003367 +deepseek-r1-distill-qwen-1.5b,FP16,32,32.1,29.8,33.55704698 +lfm2-24b-a2b,INT8-CW,32,168.9,29.8,33.55704698 +whisper-large-v3,FP16,32,177.7,30,33.33333333 +deepseek-r1-distill-qwen-1.5b,FP16,1024,89,30.1,33.22259136 +qwen2.5-1.5b-instruct,FP16,1024,90,30.1,33.22259136 +qwen2.5-coder-1.5b-instruct,FP16,1024,89.9,30.1,33.22259136 +whisper-large-v3,FP16,1024,234.8,30.1,33.22259136 +lfm2-24b-a2b,INT8-CW,1024,478.7,30.2,33.11258278 +smollm3-3b,INT8-CW,32,35.1,30.4,32.89473684 +qwen2.5-coder-3b-instruct,INT8-CW,32,36.9,30.6,32.67973856 +smollm3-3b,INT8-CW,1024,140.4,31.1,32.15434084 +minicpm3-4b,INT4-MIXED,1024,476,31.2,32.05128205 +qwen2.5-coder-3b-instruct,INT8-CW,1024,138.8,31.3,31.94888179 +trinity-mini,INT8-CW,32,174.7,31.8,31.44654088 +minicpm3-4b,INT4-MIXED,1024,488.1,32.1,31.15264798 +llama-3.2-3b-instruct,INT8-CW,32,35.8,32.3,30.95975232 +codellama-7b,INT4-MIXED,32,38.4,32.5,30.76923077 +minicpm3-4b,INT4-MIXED,1024,527.7,32.5,30.76923077 +llama-2-7b-chat-hf,INT4-MIXED,32,38.5,32.6,30.67484663 +qwen3-vl-4b-instruct,INT4-MIXED,4907,3681.6,32.8,30.48780488 +trinity-mini,INT8-CW,1024,471.8,32.8,30.48780488 +qwen3-vl-4b-instruct,INT4-MIXED,4937,3830.8,32.9,30.39513678 +llama-3.2-3b-instruct,INT8-CW,1024,136.9,33.2,30.12048193 +phi-4-multimodal-instruct,INT4-MIXED,578,315.5,33.3,30.03003003 +phi-4-multimodal-instruct,INT4-MIXED,786,372.3,33.6,29.76190476 +qwen3-vl-4b-instruct,INT4-MIXED,4907,3716.7,33.6,29.76190476 +qwen3-vl-4b-instruct,INT4-MIXED,4937,3860.1,33.6,29.76190476 +llama-2-7b-chat-hf,INT4-MIXED,32,41.7,33.7,29.6735905 +qwen3-vl-4b-instruct,INT4-MIXED,4907,3860.4,33.8,29.58579882 +qwen3-vl-4b-instruct,INT4-MIXED,4937,4022.4,33.9,29.49852507 +codellama-7b,INT4-MIXED,32,42.4,34,29.41176471 +stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,34.5,34,29.41176471 +ltx-video,INT4-MIXED,11,34.8,34.1,29.3255132 +codellama-7b,INT4-MIXED,1024,236.9,34.3,29.15451895 +llama-2-7b-chat-hf,INT4-MIXED,1024,236.6,34.3,29.15451895 +biomistral-7b-slerp,INT4-MIXED,7,36.4,34.6,28.9017341 +mistral-7b-instruct-v0.2,INT4-MIXED,32,41,34.6,28.9017341 +neural-chat-7b-v3-3,INT4-MIXED,32,41.2,34.6,28.9017341 +phi-4-multimodal-instruct,INT4-MIXED,1362,719.8,34.6,28.9017341 +zephyr-7b-beta,INT4-MIXED,32,41.2,34.6,28.9017341 +falcon-7b-instruct,INT4-MIXED,32,40.7,34.7,28.8184438 +mistral-7b-instruct-v0.3,INT4-MIXED,32,41.3,34.7,28.8184438 +phi-4-multimodal-instruct,INT4-MIXED,1570,789,34.8,28.73563218 +stable-diffusion-xl-1.0-inpainting-0.1,INT8-CW,32,35.3,34.9,28.65329513 +llama-2-7b-chat-hf,INT4-MIXED,1024,272,35.4,28.24858757 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,248.2,35.8,27.93296089 +mistral-7b-instruct-v0.2,INT4-MIXED,32,44.5,35.8,27.93296089 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,248.5,35.8,27.93296089 +mistral-7b-instruct-v0.3,INT4-MIXED,32,44.5,35.8,27.93296089 +codellama-7b,INT4-MIXED,1024,285.8,35.9,27.8551532 +zephyr-7b-beta,INT4-MIXED,1024,248,35.9,27.8551532 +falcon-7b-instruct,INT4-MIXED,1024,258.9,36,27.77777778 +neural-chat-7b-v3-3,INT4-MIXED,1024,246.8,36,27.77777778 +biomistral-7b-slerp,INT4-MIXED,7,38.5,36.2,27.62430939 +neural-chat-7b-v3-3,INT4-MIXED,32,45.4,36.2,27.62430939 +mistral-7b-instruct-v0.1,INT4-MIXED,32,45.5,36.3,27.54820937 +mistral-7b-instruct-v0.2,INT4-MIXED,32,45.3,36.3,27.54820937 +mistral-7b-instruct-v0.3,INT4-MIXED,32,45.9,36.3,27.54820937 +qwen2.5-7b-instruct,INT4-MIXED,32,44.7,36.4,27.47252747 +qwen2.5-7b-instruct-1m,INT4-MIXED,32,42.8,36.4,27.47252747 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,260.5,36.9,27.100271 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,261.5,37,27.02702703 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,63.8,37.3,26.80965147 +minicpm-v-2_6,INT4-MIXED,228,317.2,37.4,26.73796791 +qwen2.5-7b-instruct,INT4-MIXED,1024,246.8,37.4,26.73796791 +qwen2.5-7b-instruct,INT4-MIXED,32,45.5,37.4,26.73796791 +qwen2.5-7b-instruct-1m,INT4-MIXED,1024,245.8,37.4,26.73796791 +mistral-7b-instruct-v0.2,INT4-MIXED,1024,307.5,37.5,26.66666667 +mistral-7b-instruct-v0.3,INT4-MIXED,1024,307.7,37.5,26.66666667 +qwen2-7b-instruct,INT4-MIXED,32,45.6,37.5,26.66666667 +deepseek-r1-distill-qwen-7b,INT4-MIXED,32,45.2,37.6,26.59574468 +minicpm-o-2_6,INT4-MIXED,238,312.3,37.6,26.59574468 +mistral-7b-instruct-v0.1,INT4-MIXED,1025,334.4,37.6,26.59574468 +neural-chat-7b-v3-3,INT4-MIXED,1024,308.9,37.6,26.59574468 +ltx-video,INT8-CW,11,38.3,37.8,26.45502646 +deepseek-r1-distill-qwen-7b,INT4-MIXED,32,46.1,37.9,26.38522427 +qwen2-7b-instruct,INT4-MIXED,32,46.2,37.9,26.38522427 +qwen2.5-7b-instruct,INT4-MIXED,32,46.1,37.9,26.38522427 +qwen2.5-7b-instruct-1m,INT4-MIXED,32,46.3,37.9,26.38522427 +falcon-7b-instruct,INT4-MIXED,32,47.7,38.1,26.24671916 +llama-3.1-8b-instruct,INT4-MIXED,32,44.7,38.1,26.24671916 +deepseek-r1-distill-llama-8b,INT4-MIXED,32,44.8,38.3,26.10966057 +llama-3-8b-instruct,INT4-MIXED,32,45.4,38.4,26.04166667 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,58.6,38.4,26.04166667 +minicpm-v-2_6,INT4-MIXED,228,320.3,38.5,25.97402597 +qwen2-7b-instruct,INT4-MIXED,1024,255.6,38.5,25.97402597 +qwen2.5-7b-instruct,INT4-MIXED,1024,254.2,38.5,25.97402597 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,288.8,38.5,25.97402597 +deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,255.2,38.6,25.90673575 +lcm-dreamshaper-v7,INT8-HYBRID,32,39.7,38.6,25.90673575 +bloomz-7b1,INT4-MIXED,32,45.9,38.7,25.83979328 +lcm-dreamshaper-v7,INT8-HYBRID,1024,39.8,38.7,25.83979328 +deepseek-r1-distill-qwen-7b,INT4-MIXED,1024,310.3,38.8,25.77319588 +fara-7b,INT4-MIXED,32,75.3,38.9,25.70694087 +minicpm4-8b,INT4-MIXED,32,47.6,38.9,25.70694087 +qwen2-7b-instruct,INT4-MIXED,1024,310.3,38.9,25.70694087 +qwen2.5-7b-instruct,INT4-MIXED,1024,311.4,38.9,25.70694087 +qwen2.5-7b-instruct-1m,INT4-MIXED,1024,311.5,38.9,25.70694087 +minicpm-v-2_6,INT4-MIXED,228,334.5,39,25.64102564 +phi-4-mini-instruct,INT8-CW,32,44.2,39.1,25.57544757 +phi-4-mini-reasoning,INT8-CW,32,44.6,39.1,25.57544757 +qwen3-4b,INT8-CW,32,44.6,39.1,25.57544757 +llama-3.1-8b-instruct,INT4-MIXED,32,48.4,39.2,25.51020408 +minicpm-o-2_6,INT4-MIXED,238,326.7,39.2,25.51020408 +llama-3-8b-instruct,INT4-MIXED,32,47.9,39.3,25.44529262 +llama-3.1-8b-instruct,INT4-MIXED,1024,251.1,39.3,25.44529262 +qwen3-8b,INT4-MIXED,32,46.4,39.3,25.44529262 +deepseek-r1-distill-llama-8b,INT4-MIXED,1024,251.3,39.4,25.38071066 +deepseek-r1-distill-llama-8b,INT4-MIXED,32,48.2,39.4,25.38071066 +falcon-7b-instruct,INT4-MIXED,1024,333.1,39.4,25.38071066 +gemma-3-4b-it,INT8-CW,32,51.5,39.4,25.38071066 +llama-3-8b-instruct,INT4-MIXED,32,48.4,39.4,25.38071066 +qwen2.5-vl-7b-instruct,INT4-MIXED,32,59.9,39.4,25.38071066 +llama-3-8b-instruct,INT4-MIXED,1024,256.5,39.5,25.3164557 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,300.4,39.5,25.3164557 +llama-3-8b-instruct,INT4-MIXED,32,49.1,39.8,25.12562814 +fara-7b,INT4-MIXED,1024,459.7,39.9,25.06265664 +phi-4-mini-instruct,INT8-CW,1024,206.2,40,25 +qwen2.5-vl-7b-instruct,INT4-MIXED,1024,352.7,40,25 +bloomz-7b1,INT4-MIXED,32,49.8,40.1,24.93765586 +phi-4-mini-reasoning,INT8-CW,1024,206.2,40.1,24.93765586 +minicpm4-8b,INT4-MIXED,1024,279,40.2,24.87562189 +minicpm4-8b,INT4-MIXED,32,50.5,40.2,24.87562189 +bloomz-7b1,INT4-MIXED,1024,329.1,40.4,24.75247525 +qwen3-4b,INT8-CW,1024,189.3,40.4,24.75247525 +qwen3-8b,INT4-MIXED,32,49.1,40.4,24.75247525 +deepseek-r1-distill-llama-8b,INT4-MIXED,1024,265.4,40.5,24.69135802 +llama-3.1-8b-instruct,INT4-MIXED,1024,297.5,40.5,24.69135802 +minicpm-v-4_5,INT4-MIXED,217,327.8,40.5,24.69135802 +llama-3-8b-instruct,INT4-MIXED,1024,265.5,40.6,24.63054187 +llama-3-8b-instruct,INT4-MIXED,1024,265.4,40.6,24.63054187 +qwen3-8b,INT4-MIXED,1024,264.3,40.6,24.63054187 +minicpm4-8b,INT4-MIXED,32,51.9,40.7,24.57002457 +gemma-3-4b-it,INT8-CW,1024,218.4,40.9,24.44987775 +qwen3-8b,INT4-MIXED,32,50.2,40.9,24.44987775 +llama-3-8b-instruct,INT4-MIXED,1024,312.3,41,24.3902439 +afm-4.5b,INT8-CW,32,45.7,41.1,24.33090024 +gemma-4-e2b-it,FP4-NORMALIZED,274,195.1,41.5,24.09638554 +minicpm4-8b,INT4-MIXED,1024,299.1,41.6,24.03846154 +phi-3.5-vision-instruct,INT8-CW,802,278,41.7,23.98081535 +afm-4.5b,INT8-CW,1024,183,41.8,23.92344498 +bloomz-7b1,INT4-MIXED,1024,388.6,41.8,23.92344498 +qwen3-8b,INT4-MIXED,1024,277.2,41.8,23.92344498 +zephyr-7b-beta,INT4-MIXED,32,51.3,41.9,23.86634845 +minicpm4-8b,INT4-MIXED,1024,341.3,42.1,23.75296912 +gemma-4-e2b-it,FP4-NORMALIZED,1024,342,42.2,23.69668246 +minicpm-v-4_5,INT4-MIXED,217,353.2,42.2,23.69668246 +qwen3-8b,INT4-MIXED,1024,318.8,42.4,23.58490566 +phi-3.5-vision-instruct,INT8-CW,1032,338.5,42.6,23.4741784 +zephyr-7b-beta,INT4-MIXED,1024,267.5,43.2,23.14814815 +gpt-oss-20b,INT8-CW,32,187.2,43.3,23.09468822 +qwen3.5-9b,INT4-MIXED,83,109.2,43.4,23.04147465 +qwen3_8b_draft_eagle3,INT4-MIXED,32,53,43.4,23.04147465 +videochat-flash-qwen2_5-7b_internvideo2-1b,INT4-MIXED,14531,26346.6,43.5,22.98850575 +gpt-oss-20b,INT8-CW,1024,592.7,44,22.72727273 +qwen3.5-9b,INT4-MIXED,1024,446.6,44,22.72727273 +gemma-7b-it,INT4-MIXED,32,53.2,44.2,22.62443439 +qwen3_8b_draft_eagle3,INT4-MIXED,1024,319.7,44.3,22.57336343 +qwen3.5-9b,INT4-MIXED,83,114.2,45.1,22.172949 +gemma-4-e2b-it,FP16,274,193.2,45.3,22.07505519 +gemma-7b-it,INT4-MIXED,1024,313.8,45.7,21.88183807 +qwen3.5-9b,INT4-MIXED,1024,505.4,45.7,21.88183807 +gemma-4-e2b-it,FP16,1024,341.3,45.8,21.83406114 +gemma-7b-it,INT4-MIXED,32,56.6,45.8,21.83406114 +phi-4-multimodal-instruct,INT8-CW,578,291.1,46.7,21.41327623 +minicpm3-4b,INT8-CW,32,120.4,46.9,21.32196162 +phi-4-multimodal-instruct,INT8-CW,786,344.1,47,21.27659574 +deepseek-r1-distill-llama-8b,INT4-MIXED,32,61.1,47.1,21.23142251 +lcm-dreamshaper-v7,INT8-CW,32,47.7,47.1,21.23142251 +llama-3.1-8b-instruct,INT4-MIXED,32,61.4,47.1,21.23142251 +lcm-dreamshaper-v7,INT8-CW,1024,48.8,47.3,21.14164905 +gemma-7b-it,INT4-MIXED,1024,380.7,47.4,21.09704641 +lcm-dreamshaper-v7,INT8-CW,32,48.1,47.4,21.09704641 +lcm-dreamshaper-v7,INT8-CW,1024,48.8,47.6,21.00840336 +phi-4-multimodal-instruct,INT8-CW,1362,660.6,47.9,20.87682672 +phi-4-multimodal-instruct,INT8-CW,1570,716.5,48.1,20.79002079 +llama-3.1-8b-instruct,INT4-MIXED,1024,357,48.3,20.70393375 +deepseek-r1-distill-llama-8b,INT4-MIXED,1024,360.5,48.5,20.6185567 +lcm-dreamshaper-v7,FP16,1024,49.5,49,20.40816327 +lcm-dreamshaper-v7,FP16,32,49.5,49,20.40816327 +gemma-2-9b-it,INT4-MIXED,32,60.2,49.6,20.16129032 +phi-2,FP16,32,53.8,50.7,19.72386588 +gemma-2-9b-it,INT4-MIXED,32,63.8,51,19.60784314 +gemma-2-9b-it,INT4-MIXED,1024,364.2,51.3,19.49317739 +gemma-2-9b-it,INT4-MIXED,32,64.8,51.5,19.41747573 +minicpm3-4b,INT8-CW,1024,506.2,51.6,19.37984496 +gemma-2-9b-it,INT4-MIXED,1024,378.9,52.8,18.93939394 +stable-diffusion-xl-1.0-inpainting-0.1,FP16,32,53.5,52.9,18.90359168 +gemma-2-9b-it,INT4-MIXED,1024,447.7,53.3,18.76172608 +phi-2,FP16,1024,215.5,53.6,18.65671642 +ltx-video,FP16,11,54.5,53.9,18.5528757 +llama-3.2-3b-instruct,FP4-NORMALIZED,32,59.2,56.1,17.82531194 +llama-3.2-3b-instruct,FP4-NORMALIZED,1024,192.2,57,17.54385965 +smollm3-3b,FP16,32,60.4,57,17.54385965 +qwen2.5-coder-3b-instruct,FP16,32,60.8,57.3,17.45200698 +smollm3-3b,FP16,1024,176.9,57.7,17.33102253 +qwen2.5-coder-3b-instruct,FP16,1024,177.4,57.8,17.30103806 +dolly-v2-12b,INT4-MIXED,32,72.8,59.4,16.83501684 +llama-3.2-3b-instruct,FP16,32,62.4,59.7,16.75041876 +llama-3.2-3b-instruct,FP16,1024,195,61,16.39344262 +llama-2-13b-chat-hf,INT4-MIXED,32,70.2,61.3,16.3132137 +codellama-7b,INT8-CW,32,67.6,61.7,16.20745543 +llama-2-7b-chat-hf,INT8-CW,32,67.4,61.7,16.20745543 +dolly-v2-12b,INT4-MIXED,1024,661.1,61.9,16.15508885 +qwen3_8b_draft_eagle3,INT8-CW,1024,312.1,62,16.12903226 +gemma-3-12b-it,INT4-MIXED,32,85.7,63.3,15.79778831 +qwen3_8b_draft_eagle3,INT8-CW,32,80.8,63.6,15.72327044 +llama-2-13b-chat-hf,INT4-MIXED,1024,458,64,15.625 +mixtral-8x7b-v0.1,INT4-MIXED,32,1153.6,64.1,15.60062402 +llama-2-7b-chat-hf,INT8-CW,1024,273.9,64.3,15.55209953 +codellama-7b,INT8-CW,1024,272.3,64.4,15.52795031 +falcon-7b-instruct,INT8-CW,32,69.9,64.5,15.50387597 +mixtral-8x7b-v0.1,INT4-MIXED,1024,1099.5,65.4,15.29051988 +biomistral-7b-slerp,INT8-CW,7,67.7,65.5,15.26717557 +falcon-7b-instruct,INT8-CW,1024,390.4,65.5,15.26717557 +mistral-7b-instruct-v0.1,INT8-CW,32,72.5,65.6,15.24390244 +mistral-7b-instruct-v0.2,INT8-CW,32,72,65.6,15.24390244 +mistral-7b-instruct-v0.3,INT8-CW,32,72.5,65.6,15.24390244 +zephyr-7b-beta,INT8-CW,32,72.4,65.6,15.24390244 +gemma-3-12b-it,INT4-MIXED,32,85.8,65.9,15.17450683 +neural-chat-7b-v3-3,INT8-CW,32,72.2,65.9,15.17450683 +phi-4-mini-instruct,FP4-NORMALIZED,32,70.9,66.1,15.12859304 +phi-4-mini-reasoning,FP4-NORMALIZED,32,71.2,66.2,15.10574018 +qwen2-7b-instruct,INT8-CW,32,72.4,66.3,15.08295626 +qwen2.5-7b-instruct,INT8-CW,32,72.5,66.5,15.03759398 +deepseek-r1-distill-qwen-7b,INT8-CW,32,72.4,66.6,15.01501502 +qwen2.5-7b-instruct-1m,INT8-CW,32,72.6,66.7,14.99250375 +mistral-7b-instruct-v0.3,INT8-CW,1024,294.7,66.8,14.97005988 +mistral-7b-instruct-v0.1,INT8-CW,1025,318.6,66.9,14.94768311 +mistral-7b-instruct-v0.2,INT8-CW,1024,296.1,66.9,14.94768311 +qwen2-7b-instruct,INT8-CW,1024,292.1,67,14.92537313 +zephyr-7b-beta,INT8-CW,1024,294.5,67,14.92537313 +phi-4-mini-instruct,FP4-NORMALIZED,1024,281,67.1,14.90312966 +deepseek-r1-distill-qwen-7b,INT8-CW,1024,290.3,67.2,14.88095238 +gemma-3-12b-it,INT4-MIXED,1024,538.3,67.2,14.88095238 +neural-chat-7b-v3-3,INT8-CW,1024,292.9,67.2,14.88095238 +phi-4-mini-reasoning,FP4-NORMALIZED,1024,266.6,67.2,14.88095238 +qwen2.5-7b-instruct,INT8-CW,1024,291.1,67.2,14.88095238 +minicpm-v-2_6,INT8-CW,228,334.7,67.3,14.85884101 +minicpm-o-2_6,INT8-CW,238,338.8,67.5,14.81481481 +gemma-3-4b-it,FP4-NORMALIZED,32,73.1,67.6,14.79289941 +qwen2.5-7b-instruct-1m,INT8-CW,1024,292.5,67.6,14.79289941 +qwen2.5-vl-7b-instruct,INT8-CW,32,86.8,67.6,14.79289941 +fara-7b,INT8-CW,32,102,67.7,14.77104874 +lcm-sdxl,INT8-HYBRID,1024,68.6,68,14.70588235 +lcm-sdxl,INT8-HYBRID,32,68.5,68,14.70588235 +bloomz-7b1,INT8-CW,32,72.8,68.1,14.68428781 +phi-4,INT4-MIXED,32,80.7,68.4,14.61988304 +phi-4-reasoning,INT4-MIXED,32,79.1,68.4,14.61988304 +qwen2.5-vl-7b-instruct,INT8-CW,1024,336.5,68.5,14.59854015 +fara-7b,INT8-CW,1024,423.7,68.7,14.55604076 +gemma-3-4b-it,FP4-NORMALIZED,1024,277.7,69,14.49275362 +llama-3.1-8b-instruct,INT8-CW,32,76,69.1,14.47178003 +qwen1.5-14b-chat,INT4-MIXED,32,81.2,69.2,14.45086705 +deepseek-r1-distill-llama-8b,INT8-CW,32,75.8,69.3,14.43001443 +lcm-sdxl,INT8-CW,32,69.9,69.3,14.43001443 +llama-3-8b-instruct,INT8-CW,32,75.5,69.3,14.43001443 +phi-3-mini-4k-instruct,FP16,32,72.9,69.3,14.43001443 +phi-3.5-mini-instruct,FP16,32,72.8,69.3,14.43001443 +lcm-sdxl,INT8-CW,1024,70.1,69.4,14.4092219 +phi-3-mini-128k-instruct,FP16,32,73,69.4,14.4092219 +gemma-3-12b-it,INT4-MIXED,1024,608.1,69.8,14.32664756 +lcm-sdxl,INT8-CW,1024,70.7,70.1,14.26533524 +phi-4-reasoning,INT4-MIXED,1024,518.6,70.1,14.26533524 +stable-diffusion-v1-5,INT8-HYBRID,1024,71.7,70.1,14.26533524 +phi-4,INT4-MIXED,1024,523.7,70.2,14.24501425 +qwen3-8b,INT8-CW,32,77.9,70.2,14.24501425 +lcm-sdxl,INT8-CW,32,71,70.4,14.20454545 +llama-3-8b-instruct,INT8-CW,1024,296.1,70.4,14.20454545 +llama-3.1-8b-instruct,INT8-CW,1024,297.5,70.4,14.20454545 +stable-diffusion-v1-5,INT8-HYBRID,32,71.3,70.4,14.20454545 +deepseek-r1-distill-llama-8b,INT8-CW,1024,297.5,70.5,14.18439716 +phi-4-reasoning,INT4-MIXED,32,83.7,70.6,14.16430595 +bloomz-7b1,INT8-CW,1024,401.2,70.7,14.14427157 +llava-v1.6-mistral-7b-hf,INT8-CW,2944,1588.3,71.1,14.06469761 +minicpm-v-4_5,INT8-CW,217,352.5,71.3,14.02524544 +phi-4,INT4-MIXED,32,85.8,71.4,14.00560224 +phi-4-mini-reasoning,FP16,32,76.3,71.6,13.96648045 +qwen3-8b,INT8-CW,1024,306.1,71.6,13.96648045 +phi-4-mini-instruct,FP16,32,76.3,71.7,13.94700139 +qwen1.5-14b-chat,INT4-MIXED,1024,573.9,71.9,13.90820584 +deepseek-r1-distill-qwen-14b,INT4-MIXED,32,86.4,72.4,13.8121547 +phi-4-reasoning,INT4-MIXED,1024,558.5,72.5,13.79310345 +phi-3.5-vision-instruct,FP16,802,356.8,72.6,13.77410468 +phi-3.5-mini-instruct,FP16,1024,274.2,72.9,13.71742112 +videochat-flash-qwen2_5-7b_internvideo2-1b,INT8-CW,14531,26356.7,72.9,13.71742112 +phi-3-mini-4k-instruct,FP16,1024,272.6,73,13.69863014 +phi-3-mini-128k-instruct,FP16,1024,274.4,73.1,13.67989056 +gemma-3-4b-it,FP16,32,79,73.2,13.66120219 +phi-4-mini-instruct,FP16,1024,274.1,73.2,13.66120219 +phi-4-mini-reasoning,FP16,1024,276.8,73.2,13.66120219 +phi-3.5-vision-instruct,FP16,1032,452.4,73.4,13.6239782 +phi-4,INT4-MIXED,1024,645.9,73.4,13.6239782 +sd-turbo,INT8-HYBRID,32,74.9,74,13.51351351 +sd-turbo,INT8-HYBRID,1024,74.9,74.1,13.49527665 +stable-diffusion-v2-1,INT8-HYBRID,32,74.8,74.1,13.49527665 +stable-diffusion-v2-1,INT8-HYBRID,1024,74.9,74.3,13.4589502 +phi-4-reasoning,INT4-MIXED,32,98.3,74.4,13.44086022 +qwen3.5-9b,INT8-CW,83,135.3,74.5,13.42281879 +deepseek-r1-distill-qwen-14b,INT4-MIXED,1024,615.5,74.6,13.40482574 +gemma-3-4b-it,FP16,1024,282.4,74.6,13.40482574 +minicpm4-8b,INT8-CW,32,81.3,74.7,13.38688086 +qwen3-4b,FP16,32,78.8,74.7,13.38688086 +qwen3.5-9b,INT8-CW,1024,482.6,75.1,13.31557923 +minicpm4-8b,INT8-CW,1024,360.3,75.5,13.24503311 +llama-2-13b-chat-hf,INT4-MIXED,32,94.2,75.9,13.17523057 +phi-4-reasoning,INT4-MIXED,1024,722.3,76.2,13.12335958 +starcoder,INT4-MIXED,32,89.3,76.4,13.08900524 +qwen3-4b,FP16,1024,253.7,76.5,13.07189542 +llama-2-13b-chat-hf,INT4-MIXED,1024,568.9,78.6,12.72264631 +starcoder,INT4-MIXED,1024,704.6,78.6,12.72264631 +afm-4.5b,FP16,32,83.4,80.2,12.46882793 +minicpm3-4b,FP4-NORMALIZED,32,103.6,80.7,12.39157373 +starcoder2-15b,INT4-MIXED,32,104.9,80.8,12.37623762 +afm-4.5b,FP16,1024,255.2,81.1,12.33045623 +gemma-7b-it,INT8-CW,32,89.3,81.1,12.33045623 +phi-4-multimodal-instruct,FP16,578,344.9,82.1,12.18026797 +phi-4-multimodal-instruct,FP16,786,401,82.5,12.12121212 +stable-diffusion-v1-5,INT8-CW,32,83.4,82.6,12.10653753 +stable-diffusion-v1-5,INT8-CW,1024,85.2,82.7,12.09189843 +minicpm3-4b,FP16,32,102.5,82.8,12.07729469 +stable-diffusion-v1-5,INT8-CW,1024,83.6,83.1,12.03369434 +stable-diffusion-v1-5,INT8-CW,32,83.9,83.1,12.03369434 +starcoder2-15b,INT4-MIXED,1024,851.5,83.2,12.01923077 +phi-4-multimodal-instruct,FP16,1362,791.9,83.4,11.99040767 +gemma-7b-it,INT8-CW,1024,385,83.8,11.93317422 +phi-4-multimodal-instruct,FP16,1570,871.9,83.8,11.93317422 +sd-turbo,INT8-CW,32,84.7,84,11.9047619 +stable-diffusion-v2-1,INT8-CW,32,84.3,84,11.9047619 +stable-diffusion-v2-1,INT8-CW,1024,84.7,84.1,11.89060642 +qwen3-vl-4b-instruct,FP16,4937,4621.8,84.5,11.83431953 +stable-diffusion-v1-5,FP16,32,85.1,84.5,11.83431953 +qwen3-vl-4b-instruct,FP16,4907,4396.1,84.6,11.82033097 +stable-diffusion-v2-1,INT8-CW,32,85.1,84.6,11.82033097 +sd-turbo,INT8-CW,32,85.4,84.7,11.80637544 +stable-diffusion-v1-5,FP16,1024,84.8,84.7,11.80637544 +stable-diffusion-v2-1,INT8-CW,1024,85.1,84.8,11.79245283 +sd-turbo,INT8-CW,1024,85.4,85,11.76470588 +sd-turbo,INT8-CW,1024,84.4,85,11.76470588 +minicpm3-4b,FP4-NORMALIZED,1024,561.9,85.5,11.69590643 +stable-diffusion-v2-1,FP16,32,86,85.7,11.66861144 +sd-turbo,FP16,32,85.6,86.1,11.61440186 +stable-diffusion-v2-1,FP16,1024,85.7,86.3,11.58748552 +sd-turbo,FP16,1024,85.8,86.6,11.54734411 +gemma-2-9b-it,INT8-CW,32,96.9,88.2,11.33786848 +lcm-sdxl,FP16,1024,89,88.5,11.29943503 +lcm-sdxl,FP16,32,88.9,88.5,11.29943503 +minicpm3-4b,FP16,1024,570.2,90,11.11111111 +gemma-2-9b-it,INT8-CW,1024,428.3,90.6,11.03752759 \ No newline at end of file diff --git a/docs/sphinx_setup/_static/download/benchmark_genai_platform_list.pdf b/docs/sphinx_setup/_static/download/benchmark_genai_platform_list.pdf index c3c96dff08161f..7b3fa125f04011 100644 Binary files a/docs/sphinx_setup/_static/download/benchmark_genai_platform_list.pdf and b/docs/sphinx_setup/_static/download/benchmark_genai_platform_list.pdf differ