[GGUF] Use the OpenVINO GGUF frontend: tokenizer from rt_info, stateful graph

.github/workflows/linux.yml

@@ -680,6 +680,23 @@ jobs:
               python -m pytest -v ./tools/who_what_benchmark/tests -m "transformers_dependent or transformers_lower_v5" -k "not videochat"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
             timeout: 120
+          # GGUF accuracy: WWB compares GenAI loading a .gguf through the OpenVINO GGUF frontend
+          # against llama.cpp running the same file, so a conversion regression (wrong layer wiring,
+          # bad KV-cache precision, broken tokenizer) shows up as a similarity drop rather than
+          # silently producing plausible-looking text.
+          #
+          # The suite self-skips unless WWB_GGUF_TESTS=1 because it downloads real models; `-m
+          # gguf_small` further restricts it to the sub-1 GB fixtures (llama / qwen2 / qwen3 /
+          # hunyuan-dense), the ones that fit a CI runner's disk and time budget. The larger
+          # architecture fixtures stay an opt-in local run.
+          #
+          # The job name must start with "WWB" -- that is what gates the llama-cpp-python install
+          # step below, and llama.cpp is the reference side of this comparison.
+          - name: 'WWB tests (GGUF)'
+            cmd: |
+              WWB_GGUF_TESTS=1 python -m pytest -v ./tools/who_what_benchmark/tests/test_cli_text_gguf.py -m gguf_small
+            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).GGUF.test || fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
+            timeout: 120
           - name: 'EAGLE3 speculative decoding tests'
             cmd: |
               python -m pip install transformers==4.57.6

CMakeLists.txt

@@ -98,6 +98,9 @@ endif()
 if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tools/continuous_batching" AND ENABLE_TOOLS)
     add_subdirectory(tools/continuous_batching)
 endif()
+if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tools/gguf_arch_check" AND ENABLE_TOOLS)
+    add_subdirectory(tools/gguf_arch_check)
+endif()
 if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tests/cpp" AND ENABLE_TESTS)
     add_subdirectory(tests/cpp)
 endif()

samples/cpp/text_generation/README.md

@@ -27,8 +27,17 @@ hf download <model> --local-dir <output_folder>
 
 To run any samples with a GGUF model, simply provide the path to the .gguf file via the `<MODEL_DIR>` parameter.
 
-This capability is currently available in preview mode and supports a limited set of topologies, including SmolLM and Qwen2.5. For other models
-and architectures, we still recommend converting the model to the IR format using the `optimum-intel` tool.
+GGUF files are converted by the OpenVINO GGUF frontend, which covers a wide range of architectures.
+For anything it does not accept, convert the model to the IR format with the `optimum-intel` tool.
+
+GGUF models run on the SDPA attention backend; continuous batching / PagedAttention is not
+supported for them, and a `scheduler_config` passed alongside one is ignored.
+
+> [!NOTE]
+> Setting `OPENVINO_GENAI_GGUF_LEGACY_READER=1` converts the file with the older, pre-frontend
+> reader instead. It exists only as a temporary fallback and will be removed together with that
+> code: it handles just `llama`, `qwen2` and `qwen3`, and it ignores part of the file's metadata
+> (for example `rope_freqs.weight`, so llama-3 RoPE scaling is not applied and accuracy suffers).
 
 ## Sample Descriptions
 ### Common information

src/cpp/CMakeLists.txt

@@ -230,6 +230,18 @@ endif()
 if(ENABLE_GGUF)
     target_link_libraries(${TARGET_NAME_OBJ} PRIVATE gguflib)
     target_compile_definitions(${TARGET_NAME_OBJ} PRIVATE ENABLE_GGUF)
+    # GGUF conversion routes through the OpenVINO GGUF frontend + its AdaptToGenAI pass
+    # (header openvino/frontend/gguf/adapt_to_genai.hpp). The frontend is a LINKABLE_FRONTEND
+    # exported by OpenVINO as openvino::frontend::gguf (legacy name: openvino::frontend::ggml).
+    if(TARGET openvino::frontend::gguf)
+        target_link_libraries(${TARGET_NAME_OBJ} PRIVATE openvino::frontend::gguf)
+    elseif(TARGET openvino::frontend::ggml)
+        target_link_libraries(${TARGET_NAME_OBJ} PRIVATE openvino::frontend::ggml)
+    else()
+        message(FATAL_ERROR "ENABLE_GGUF=ON requires the OpenVINO GGUF frontend "
+                            "(openvino::frontend::gguf). Build OpenVINO with "
+                            "-DENABLE_OV_GGUF_FRONTEND=ON.")
+    endif()
 endif()
 
 target_include_directories(${TARGET_NAME_OBJ} SYSTEM PRIVATE "${safetensors.h_SOURCE_DIR}")
@@ -292,6 +304,11 @@ endif()
 
 if(ENABLE_GGUF)
   target_link_libraries(${TARGET_NAME} PRIVATE gguflib)
+  if(TARGET openvino::frontend::gguf)
+    target_link_libraries(${TARGET_NAME} PRIVATE openvino::frontend::gguf)
+  elseif(TARGET openvino::frontend::ggml)
+    target_link_libraries(${TARGET_NAME} PRIVATE openvino::frontend::ggml)
+  endif()
 endif()
 
 target_compile_features(${TARGET_NAME} INTERFACE cxx_std_17)

src/cpp/include/openvino/genai/tokenizer.hpp

@@ -50,6 +50,18 @@ class OPENVINO_GENAI_EXPORTS Tokenizer {
      */
     explicit Tokenizer(const std::filesystem::path& tokenizer_path, const ov::AnyMap& properties = {});
 
+    /**
+     * @brief ov::genai::Tokenizer constructor from a GGUF-frontend model.
+     *
+     * Builds the tokenizer/detokenizer from the GGUF tokenizer metadata that the OpenVINO GGUF
+     * frontend attached to the model's runtime info, so the .gguf file is not re-opened (the
+     * language model was already read from it). The model must be one produced by reading a
+     * .gguf through the frontend; otherwise an exception is thrown.
+     * @param gguf_model model carrying GGUF tokenizer metadata in rt_info
+     * @param properties Properties passed to ov::Core::compile_model
+     */
+    explicit Tokenizer(const std::shared_ptr<ov::Model>& gguf_model, const ov::AnyMap& properties = {});
+
     /**
      * @brief ov::genai::Tokenizer constructor to initialize directly from model and weights
      *

src/cpp/src/gguf_utils/gguf.cpp

@@ -346,6 +346,13 @@ GGUFLoad get_gguf_data(const std::string& file) {
     }
 }
 
+std::unordered_map<std::string, GGUFMetaData> get_gguf_metadata(const std::string& file) {
+    check_file(file);
+    std::unique_ptr<gguf_ctx, decltype(&gguf_close)> ctx(gguf_open(file.data()), gguf_close);
+    OPENVINO_ASSERT(ctx, "Failed to open '", file, "' with gguf_open");
+    return load_metadata(ctx.get());
+}
+
 float metadata_to_float(const std::unordered_map<std::string, GGUFMetaData>& metadata, const std::string& key) {
     auto tensor = std::get<ov::Tensor>(metadata.at(key));
     return *(tensor.data<ov::element_type_traits<ov::element::f32>::value_type>());

src/cpp/src/gguf_utils/gguf.hpp

@@ -51,3 +51,7 @@ std::tuple<std::map<std::string, GGUFMetaData>,
 load_gguf(const std::string& file);
 
 GGUFLoad get_gguf_data(const std::string& file);
+
+// Load only GGUF metadata KV pairs without reading or dequantizing weight tensors.
+// Used by the tokenizer path, which needs only the metadata.
+std::unordered_map<std::string, GGUFMetaData> get_gguf_metadata(const std::string& file);

src/cpp/src/gguf_utils/gguf_modeling.cpp

@@ -11,6 +11,11 @@
 #include <openvino/openvino.hpp>
 #include "openvino/runtime/core.hpp"
 #include "openvino/opsets/opset13.hpp"
+#include "openvino/pass/manager.hpp"
+#include "openvino/frontend/extension/decoder_transformation.hpp"
+#include "openvino/frontend/gguf/adapt_to_genai.hpp"
+#include "openvino/frontend/gguf/frontend.hpp"
+#include "openvino/frontend/gguf/make_stateful.hpp"
 
 #include "gguf_utils/building_blocks.hpp"
 #include "gguf_utils/gguf_modeling.hpp"
@@ -138,37 +143,88 @@ std::shared_ptr<ov::Model> create_language_model(
 
 } // namespace
 
+// Escape hatch while the pre-frontend reader is still around: setting
+// OPENVINO_GENAI_GGUF_LEGACY_READER=1 converts .gguf with the old hand-written builder instead of
+// the GGUF frontend. It only ever supported llama / qwen2 / qwen3 and ignores some of the file's
+// metadata (notably rope_freqs.weight, so llama-3 RoPE scaling is not applied), but it is kept as
+// a fallback until that code is deleted. Remove together with create_language_model().
+bool use_legacy_gguf_reader() {
+    const char* env = std::getenv("OPENVINO_GENAI_GGUF_LEGACY_READER");
+    return env != nullptr && std::string(env) != "0";
+}
+
 std::shared_ptr<ov::Model> create_from_gguf(const std::string& model_path, const bool enable_save_ov_model) {
     auto start_time = std::chrono::high_resolution_clock::now();
     std::stringstream ss;
-    ss << "Loading and unpacking model from: " << model_path;
-    ov::genai::utils::print_gguf_debug_info(ss.str());
-    auto [config, consts, qtypes] = load_gguf(model_path);
-    auto load_finish_time = std::chrono::high_resolution_clock::now();
 
-    ss.str("");
-    ss << "Loading and unpacking model done. Time: " << std::chrono::duration_cast<std::chrono::milliseconds>(load_finish_time - start_time).count() << "ms";
-    ov::genai::utils::print_gguf_debug_info(ss.str());
-
-    std::shared_ptr<ov::Model> model;
-    const std::string model_arch = std::get<std::string>(config.at("architecture"));
-    ss.str("");
-    ss << "Start generating OpenVINO model...";
-    ov::genai::utils::print_gguf_debug_info(ss.str());
-    if (!model_arch.compare("llama") || !model_arch.compare("qwen2") || !model_arch.compare("qwen3")) {
-        model = create_language_model(config, consts, qtypes);
-        if (enable_save_ov_model){
+    if (use_legacy_gguf_reader()) {
+        ss << "OPENVINO_GENAI_GGUF_LEGACY_READER is set: converting with the legacy GGUF reader "
+              "instead of the OpenVINO GGUF frontend: "
+           << model_path;
+        ov::genai::utils::print_gguf_debug_info(ss.str());
+
+        auto [config, consts, qtypes] = load_gguf(model_path);
+        const std::string model_arch = std::get<std::string>(config.at("architecture"));
+        OPENVINO_ASSERT(model_arch == "llama" || model_arch == "qwen2" || model_arch == "qwen3",
+                        "The legacy GGUF reader does not support architecture '",
+                        model_arch,
+                        "'. Unset OPENVINO_GENAI_GGUF_LEGACY_READER to use the GGUF frontend.");
+        auto legacy_model = create_language_model(config, consts, qtypes);
+        if (enable_save_ov_model) {
             std::filesystem::path gguf_model_path(model_path);
             std::filesystem::path save_path = gguf_model_path.parent_path() / "openvino_model.xml";
-            ov::genai::utils::save_openvino_model(model, save_path.string(), true);
+            ov::genai::utils::save_openvino_model(legacy_model, save_path.string(), true);
         }
-    } else {
-        OPENVINO_THROW("Unsupported model architecture '", model_arch, "'");
+        ss.str("");
+        ss << "Legacy GGUF conversion done. Time: "
+           << std::chrono::duration_cast<std::chrono::milliseconds>(
+                  std::chrono::high_resolution_clock::now() - start_time)
+                  .count()
+           << "ms";
+        ov::genai::utils::print_gguf_debug_info(ss.str());
+        return legacy_model;
+    }
+
+    ss << "Loading and converting GGUF model via the OpenVINO GGUF frontend: " << model_path;
+    ov::genai::utils::print_gguf_debug_info(ss.str());
+
+    // Convert the GGUF file through the OpenVINO GGUF frontend, which supports a wide range of
+    // architectures (llama / qwen2 / qwen3 / phi3 / minicpm / gpt-oss / ...).
+    //
+    // The frontend is universal and always converts to a STATELESS graph -- every KV cache is an
+    // explicit input/output pair -- exactly as optimum-intel exports before it applies its own
+    // make-stateful transformation. Statefulness is the caller's choice, so we make it here, by
+    // registering ov::frontend::gguf::pass::MakeStateful as a transformation extension: the
+    // frontend runs it in its normalization stage, ahead of the built-in stateless lowering, so
+    // each cache becomes a ReadValue/Concat/Assign OpenVINO state instead.
+    //
+    // The frontend is driven directly rather than through core.read_model(model_path) so the
+    // extension is scoped to this one conversion; ov::Core::add_extension would be global.
+    //
+    // Two passes then run over the stateful graph:
+    //   - MakeStateful (above, inside conversion) -- the KV caches.
+    //   - AdaptToGenAI (below) -- the IO contract: the frontend's gguf-native inputs become the
+    //     ones genai's StatefulLLMPipeline drives (input_ids / attention_mask / position_ids /
+    //     beam_idx -> logits).
+    ov::frontend::gguf::FrontEnd frontend;
+    frontend.add_extension(std::make_shared<ov::frontend::DecoderTransformationExtension>(
+        ov::frontend::gguf::pass::MakeStateful()));
+    auto model = frontend.convert(frontend.load(model_path));
+
+    ov::pass::Manager manager;
+    manager.register_pass<ov::frontend::gguf::pass::AdaptToGenAI>();
+    manager.run_passes(model);
+
+    if (enable_save_ov_model) {
+        std::filesystem::path gguf_model_path(model_path);
+        std::filesystem::path save_path = gguf_model_path.parent_path() / "openvino_model.xml";
+        ov::genai::utils::save_openvino_model(model, save_path.string(), true);
     }
+
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-        std::chrono::high_resolution_clock::now() - load_finish_time).count();
+        std::chrono::high_resolution_clock::now() - start_time).count();
     ss.str("");
-    ss << "Model generation done. Time: " << duration << "ms";
+    ss << "GGUF model conversion done. Time: " << duration << "ms";
     ov::genai::utils::print_gguf_debug_info(ss.str());
 
     return model;

src/cpp/src/gguf_utils/gguf_modeling.hpp

@@ -8,3 +8,10 @@
 #include "openvino/openvino.hpp"
 
 std::shared_ptr<ov::Model> create_from_gguf(const std::string& model_path, const bool enable_save_ov_model);
+
+/// \brief True when OPENVINO_GENAI_GGUF_LEGACY_READER selects the pre-frontend GGUF reader.
+///
+/// Temporary fallback while the legacy reader is still in the tree. It produces a model without
+/// the frontend's tokenizer metadata in rt_info, so callers that would normally build the
+/// tokenizer from the model have to read the .gguf instead.
+bool use_legacy_gguf_reader();