[GGUF] Use the OpenVINO GGUF frontend: tokenizer from rt_info, stateful graph

.github/workflows/linux.yml

@@ -680,6 +680,23 @@ jobs:
               python -m pytest -v ./tools/who_what_benchmark/tests -m "transformers_dependent or transformers_lower_v5" -k "not videochat"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
             timeout: 120
+          # GGUF accuracy: WWB compares GenAI loading a .gguf through the OpenVINO GGUF frontend
+          # against llama.cpp running the same file, so a conversion regression (wrong layer wiring,
+          # bad KV-cache precision, broken tokenizer) shows up as a similarity drop rather than
+          # silently producing plausible-looking text.
+          #
+          # The suite self-skips unless WWB_GGUF_TESTS=1 because it downloads real models; `-m
+          # gguf_small` further restricts it to the sub-1 GB fixtures (llama / qwen2 / qwen3 /
+          # hunyuan-dense), the ones that fit a CI runner's disk and time budget. The larger
+          # architecture fixtures stay an opt-in local run.
+          #
+          # The job name must start with "WWB" -- that is what gates the llama-cpp-python install
+          # step below, and llama.cpp is the reference side of this comparison.
+          - name: 'WWB tests (GGUF)'
+            cmd: |
+              WWB_GGUF_TESTS=1 python -m pytest -v ./tools/who_what_benchmark/tests/test_cli_text_gguf.py -m gguf_small
+            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).GGUF.test || fromJSON(needs.smart_ci.outputs.affected_components).WWB.test }}
+            timeout: 120
           - name: 'EAGLE3 speculative decoding tests'
             cmd: |
               python -m pip install transformers==4.57.6

CMakeLists.txt

@@ -98,6 +98,9 @@ endif()
 if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tools/continuous_batching" AND ENABLE_TOOLS)
     add_subdirectory(tools/continuous_batching)
 endif()
+if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tools/gguf_arch_check" AND ENABLE_TOOLS)
+    add_subdirectory(tools/gguf_arch_check)
+endif()
 if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tests/cpp" AND ENABLE_TESTS)
     add_subdirectory(tests/cpp)
 endif()

samples/cpp/text_generation/README.md

@@ -27,8 +27,20 @@ hf download <model> --local-dir <output_folder>
 
 To run any samples with a GGUF model, simply provide the path to the .gguf file via the `<MODEL_DIR>` parameter.
 
-This capability is currently available in preview mode and supports a limited set of topologies, including SmolLM and Qwen2.5. For other models
-and architectures, we still recommend converting the model to the IR format using the `optimum-intel` tool.
+GGUF files are converted by the OpenVINO GGUF frontend, which covers a wide range of architectures.
+For anything it does not accept, convert the model to the IR format with the `optimum-intel` tool.
+
+GGUF models run on the SDPA attention backend; continuous batching / PagedAttention is not
+supported for them, and a `scheduler_config` passed alongside one is ignored.
+
+> [!NOTE]
+> The `GGUF_READER` property selects which reader converts the file, in the same way
+> `ATTENTION_BACKEND` selects the attention backend. It defaults to `"FRONTEND"`; passing
+> `ov::genai::gguf_reader("LEGACY")` (C++) or `GGUF_READER="LEGACY"` (Python) uses the older,
+> pre-frontend reader instead. That exists only as a temporary fallback and will be removed
+> together with that code: it handles just `llama`, `qwen2` and `qwen3`, and it ignores part of
+> the file's metadata (for example `rope_freqs.weight`, so llama-3 RoPE scaling is not applied
+> and accuracy suffers).
 
 ## Sample Descriptions
 ### Common information

src/cpp/CMakeLists.txt

@@ -230,6 +230,18 @@ endif()
 if(ENABLE_GGUF)
     target_link_libraries(${TARGET_NAME_OBJ} PRIVATE gguflib)
     target_compile_definitions(${TARGET_NAME_OBJ} PRIVATE ENABLE_GGUF)
+    # GGUF conversion routes through the OpenVINO GGUF frontend + its AdaptToGenAI pass
+    # (header openvino/frontend/gguf/adapt_to_genai.hpp). The frontend is a LINKABLE_FRONTEND
+    # exported by OpenVINO as openvino::frontend::gguf (legacy name: openvino::frontend::ggml).
+    if(TARGET openvino::frontend::gguf)
+        target_link_libraries(${TARGET_NAME_OBJ} PRIVATE openvino::frontend::gguf)
+    elseif(TARGET openvino::frontend::ggml)
+        target_link_libraries(${TARGET_NAME_OBJ} PRIVATE openvino::frontend::ggml)
+    else()
+        message(FATAL_ERROR "ENABLE_GGUF=ON requires the OpenVINO GGUF frontend "
+                            "(openvino::frontend::gguf). Build OpenVINO with "
+                            "-DENABLE_OV_GGUF_FRONTEND=ON.")
+    endif()
 endif()
 
 target_include_directories(${TARGET_NAME_OBJ} SYSTEM PRIVATE "${safetensors.h_SOURCE_DIR}")
@@ -292,6 +304,11 @@ endif()
 
 if(ENABLE_GGUF)
   target_link_libraries(${TARGET_NAME} PRIVATE gguflib)
+  if(TARGET openvino::frontend::gguf)
+    target_link_libraries(${TARGET_NAME} PRIVATE openvino::frontend::gguf)
+  elseif(TARGET openvino::frontend::ggml)
+    target_link_libraries(${TARGET_NAME} PRIVATE openvino::frontend::ggml)
+  endif()
 endif()
 
 target_compile_features(${TARGET_NAME} INTERFACE cxx_std_17)

src/cpp/include/openvino/genai/llm_pipeline.hpp

@@ -411,6 +411,22 @@ static constexpr ov::Property<bool> prompt_lookup{"prompt_lookup"};
 */
 static constexpr ov::Property<bool> enable_save_ov_model{"enable_save_ov_model"};
 
+/**
+* @brief Selects which reader converts a `.gguf` file, in the same style as `ATTENTION_BACKEND`
+* selects between the PagedAttention and SDPA backends.
+*
+* Accepted values:
+*  - `"FRONTEND"` (default) - the OpenVINO GGUF frontend, which supports the full set of
+*    architectures and is the path GenAI is built around.
+*  - `"LEGACY"` - the pre-frontend, hand-written GGUF reader. A temporary fallback that will be
+*    removed with that code. It is NOT equivalent: it only handles llama / qwen2 / qwen3, it
+*    rejects some files the frontend reads, and it ignores `rope_freqs.weight`, so llama-3 RoPE
+*    scaling is silently not applied and its output differs from the frontend's.
+*
+* Example: `LLMPipeline(model_path, "CPU", ov::genai::gguf_reader("LEGACY"))`.
+*/
+static constexpr ov::Property<std::string> gguf_reader{"GGUF_READER"};
+
 
 }  // namespace genai
 }  // namespace ov

src/cpp/include/openvino/genai/tokenizer.hpp

@@ -29,6 +29,39 @@ struct TokenizedInputs {
     std::optional<ov::Tensor> token_type_ids;
 };
 
+/**
+ * @brief The GGUF tokenizer metadata a Tokenizer can be built from: the `tokenizer.*` keys of a
+ * .gguf file.
+ *
+ * This is what the tokenizer is actually built from, so it is what the constructor asks for. An
+ * ov::Model is only ever a container the OpenVINO GGUF frontend happens to attach this to, and
+ * taking a model would suggest any model works -- one without the metadata (a plain IR, or a
+ * .gguf read by the legacy reader) cannot produce a tokenizer, and that is much better reported
+ * where the metadata is looked up than from inside a Tokenizer constructor.
+ *
+ * `config` is keyed by the metadata sub-key -- the part after the last dot of `tokenizer.ggml.*` /
+ * `tokenizer.chat_template`, e.g. "model", "tokens", "merges", "scores", "token_type", "pre",
+ * "bos_token_id", "eos_token_id", "chat_template". Each value holds one of std::string,
+ * std::vector<std::string> or ov::Tensor (arrays, and scalars as a shape-{} tensor), mirroring the
+ * GGUF metadata variant.
+ */
+struct OPENVINO_GENAI_EXPORTS GGUFTokenizerParameters {
+    /// tokenizer sub-key -> value (string / vector<string> / Tensor)
+    ov::AnyMap config;
+
+    GGUFTokenizerParameters() = default;
+    explicit GGUFTokenizerParameters(ov::AnyMap config) : config(std::move(config)) {}
+
+    /**
+     * @brief Read the parameters out of a model converted by the OpenVINO GGUF frontend, which
+     * attaches them to the model's runtime info so the .gguf need not be re-opened.
+     * @param gguf_model model produced by reading a .gguf through the GGUF frontend.
+     * @throws ov::Exception if the model is null or carries no GGUF tokenizer metadata -- e.g. a
+     *         plain IR, a model whose rt_info was stripped, or one built by the legacy GGUF reader.
+     */
+    static GGUFTokenizerParameters from_model(const std::shared_ptr<ov::Model>& gguf_model);
+};
+
 /**
  * @brief The class is used to encode prompts and decode resulting tokens
  *
@@ -50,6 +83,18 @@ class OPENVINO_GENAI_EXPORTS Tokenizer {
      */
     explicit Tokenizer(const std::filesystem::path& tokenizer_path, const ov::AnyMap& properties = {});
 
+    /**
+     * @brief ov::genai::Tokenizer constructor from GGUF tokenizer metadata.
+     *
+     * Builds the tokenizer/detokenizer from the `tokenizer.*` metadata of a .gguf file, so the
+     * file need not be re-opened when the language model was already read from it. Obtain the
+     * parameters with GGUFTokenizerParameters::from_model() for a model converted by the OpenVINO
+     * GGUF frontend, or construct them from a metadata map directly.
+     * @param gguf_tokenizer_parameters GGUF tokenizer metadata
+     * @param properties Properties passed to ov::Core::compile_model
+     */
+    explicit Tokenizer(const GGUFTokenizerParameters& gguf_tokenizer_parameters, const ov::AnyMap& properties = {});
+
     /**
      * @brief ov::genai::Tokenizer constructor to initialize directly from model and weights
      *

src/cpp/src/continuous_batching/pipeline.cpp

@@ -65,7 +65,7 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline( const std::filesystem::p
     utils::extract_extensions_to_core(properties_without_draft_model);
     auto model = utils::read_model(models_path,
         utils::get_model_properties(properties_without_draft_model, "language_model"));
-    auto [properties_without_draft_model_without_gguf, enable_save_ov_model] = utils::extract_gguf_properties(properties_without_draft_model);
+    auto properties_without_draft_model_without_gguf = utils::extract_gguf_properties(properties_without_draft_model).rest;
     properties_without_draft_model_without_gguf[ov::cache_model_path.name()] = models_path;
     auto tokenizer = ov::genai::Tokenizer(models_path, tokenizer_properties);
     auto generation_config = utils::from_config_json_if_exists(models_path);
@@ -128,7 +128,7 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(const std::shared_ptr<ov:
     utils::validate_vlm_model_properties(properties_without_draft_model);
 
     auto model = language_model;
-    auto [properties_without_draft_model_without_gguf, enable_save_ov_model] = utils::extract_gguf_properties(properties_without_draft_model);
+    auto properties_without_draft_model_without_gguf = utils::extract_gguf_properties(properties_without_draft_model).rest;
     properties_without_draft_model_without_gguf[ov::cache_model_path.name()] = models_path;
     auto tokenizer = ov::genai::Tokenizer(models_path, tokenizer_properties);
     auto generation_config = utils::from_config_json_if_exists(models_path);
@@ -191,7 +191,7 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(
     utils::extract_extensions_to_core(properties_without_draft_model);
     auto model = utils::read_model(models_path,
         utils::get_model_properties(properties_without_draft_model, "language_model"));
-    auto [properties_without_draft_model_without_gguf, enable_save_ov_model] = utils::extract_gguf_properties(properties_without_draft_model);
+    auto properties_without_draft_model_without_gguf = utils::extract_gguf_properties(properties_without_draft_model).rest;
     properties_without_draft_model_without_gguf[ov::cache_model_path.name()] = models_path;
 
     auto generation_config = utils::from_config_json_if_exists(models_path);
@@ -246,7 +246,7 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(
     auto is_prompt_lookup_enabled = extract_prompt_lookup_from_config(properties_without_draft_model);
     auto dflash_rt_info = utils::dflash::extract_dflash_info_from_config(draft_model_descr.properties);
     auto eagle_rt_info = utils::eagle3::extract_eagle3_info_from_config(draft_model_descr.properties, model_config_dir_path);
-    auto [properties_without_draft_model_without_gguf, enable_save_ov_model] = utils::extract_gguf_properties(properties_without_draft_model);
+    auto properties_without_draft_model_without_gguf = utils::extract_gguf_properties(properties_without_draft_model).rest;
     if (!model_config_dir_path.empty()) {
         properties_without_draft_model_without_gguf[ov::cache_model_path.name()] = model_config_dir_path;
     }

src/cpp/src/gguf_utils/gguf.cpp

@@ -346,6 +346,13 @@ GGUFLoad get_gguf_data(const std::string& file) {
     }
 }
 
+std::unordered_map<std::string, GGUFMetaData> get_gguf_metadata(const std::string& file) {
+    check_file(file);
+    std::unique_ptr<gguf_ctx, decltype(&gguf_close)> ctx(gguf_open(file.data()), gguf_close);
+    OPENVINO_ASSERT(ctx, "Failed to open '", file, "' with gguf_open");
+    return load_metadata(ctx.get());
+}
+
 float metadata_to_float(const std::unordered_map<std::string, GGUFMetaData>& metadata, const std::string& key) {
     auto tensor = std::get<ov::Tensor>(metadata.at(key));
     return *(tensor.data<ov::element_type_traits<ov::element::f32>::value_type>());

src/cpp/src/gguf_utils/gguf.hpp

@@ -51,3 +51,7 @@ std::tuple<std::map<std::string, GGUFMetaData>,
 load_gguf(const std::string& file);
 
 GGUFLoad get_gguf_data(const std::string& file);
+
+// Load only GGUF metadata KV pairs without reading or dequantizing weight tensors.
+// Used by the tokenizer path, which needs only the metadata.
+std::unordered_map<std::string, GGUFMetaData> get_gguf_metadata(const std::string& file);

src/cpp/src/gguf_utils/gguf_modeling.cpp

@@ -11,6 +11,11 @@
 #include <openvino/openvino.hpp>
 #include "openvino/runtime/core.hpp"
 #include "openvino/opsets/opset13.hpp"
+#include "openvino/pass/manager.hpp"
+#include "openvino/frontend/extension/decoder_transformation.hpp"
+#include "openvino/frontend/gguf/adapt_to_genai.hpp"
+#include "openvino/frontend/gguf/frontend.hpp"
+#include "openvino/frontend/gguf/make_stateful.hpp"
 
 #include "gguf_utils/building_blocks.hpp"
 #include "gguf_utils/gguf_modeling.hpp"
@@ -138,37 +143,86 @@ std::shared_ptr<ov::Model> create_language_model(
 
 } // namespace
 
-std::shared_ptr<ov::Model> create_from_gguf(const std::string& model_path, const bool enable_save_ov_model) {
+std::shared_ptr<ov::Model> create_from_gguf(const std::string& model_path,
+                                            bool enable_save_ov_model,
+                                            bool use_legacy_reader) {
     auto start_time = std::chrono::high_resolution_clock::now();
     std::stringstream ss;
-    ss << "Loading and unpacking model from: " << model_path;
-    ov::genai::utils::print_gguf_debug_info(ss.str());
-    auto [config, consts, qtypes] = load_gguf(model_path);
-    auto load_finish_time = std::chrono::high_resolution_clock::now();
-
-    ss.str("");
-    ss << "Loading and unpacking model done. Time: " << std::chrono::duration_cast<std::chrono::milliseconds>(load_finish_time - start_time).count() << "ms";
-    ov::genai::utils::print_gguf_debug_info(ss.str());
 
-    std::shared_ptr<ov::Model> model;
-    const std::string model_arch = std::get<std::string>(config.at("architecture"));
-    ss.str("");
-    ss << "Start generating OpenVINO model...";
-    ov::genai::utils::print_gguf_debug_info(ss.str());
-    if (!model_arch.compare("llama") || !model_arch.compare("qwen2") || !model_arch.compare("qwen3")) {
-        model = create_language_model(config, consts, qtypes);
-        if (enable_save_ov_model){
+    // Escape hatch while the pre-frontend reader is still around: ov::genai::gguf_reader("LEGACY")
+    // converts .gguf with the old hand-written builder instead of the GGUF frontend. It only ever
+    // supported llama / qwen2 / qwen3 and ignores some of the file's metadata (notably
+    // rope_freqs.weight, so llama-3 RoPE scaling is not applied), but it is kept as a fallback
+    // until that code is deleted. Remove together with create_language_model().
+    if (use_legacy_reader) {
+        ss << "ov::genai::gguf_reader is '" << ov::genai::LEGACY_GGUF_READER
+           << "': converting with the legacy GGUF reader instead of the OpenVINO GGUF frontend: " << model_path;
+        ov::genai::utils::print_gguf_debug_info(ss.str());
+
+        auto [config, consts, qtypes] = load_gguf(model_path);
+        const std::string model_arch = std::get<std::string>(config.at("architecture"));
+        OPENVINO_ASSERT(model_arch == "llama" || model_arch == "qwen2" || model_arch == "qwen3",
+                        "The legacy GGUF reader does not support architecture '",
+                        model_arch,
+                        "'. Set the ov::genai::gguf_reader property to '",
+                        ov::genai::FRONTEND_GGUF_READER,
+                        "' (the default) to use the GGUF frontend.");
+        auto legacy_model = create_language_model(config, consts, qtypes);
+        if (enable_save_ov_model) {
             std::filesystem::path gguf_model_path(model_path);
             std::filesystem::path save_path = gguf_model_path.parent_path() / "openvino_model.xml";
-            ov::genai::utils::save_openvino_model(model, save_path.string(), true);
+            ov::genai::utils::save_openvino_model(legacy_model, save_path.string(), true);
         }
-    } else {
-        OPENVINO_THROW("Unsupported model architecture '", model_arch, "'");
+        ss.str("");
+        ss << "Legacy GGUF conversion done. Time: "
+           << std::chrono::duration_cast<std::chrono::milliseconds>(
+                  std::chrono::high_resolution_clock::now() - start_time)
+                  .count()
+           << "ms";
+        ov::genai::utils::print_gguf_debug_info(ss.str());
+        return legacy_model;
     }
+
+    ss << "Loading and converting GGUF model via the OpenVINO GGUF frontend: " << model_path;
+    ov::genai::utils::print_gguf_debug_info(ss.str());
+
+    // Convert the GGUF file through the OpenVINO GGUF frontend, which supports a wide range of
+    // architectures (llama / qwen2 / qwen3 / phi3 / minicpm / gpt-oss / ...).
+    //
+    // The frontend is universal and always converts to a STATELESS graph -- every KV cache is an
+    // explicit input/output pair -- exactly as optimum-intel exports before it applies its own
+    // make-stateful transformation. Statefulness is the caller's choice, so we make it here, by
+    // registering ov::frontend::gguf::pass::MakeStateful as a transformation extension: the
+    // frontend runs it in its normalization stage, ahead of the built-in stateless lowering, so
+    // each cache becomes a ReadValue/Concat/Assign OpenVINO state instead.
+    //
+    // The frontend is driven directly rather than through core.read_model(model_path) so the
+    // extension is scoped to this one conversion; ov::Core::add_extension would be global.
+    //
+    // Two passes then run over the stateful graph:
+    //   - MakeStateful (above, inside conversion) -- the KV caches.
+    //   - AdaptToGenAI (below) -- the IO contract: the frontend's gguf-native inputs become the
+    //     ones genai's StatefulLLMPipeline drives (input_ids / attention_mask / position_ids /
+    //     beam_idx -> logits).
+    ov::frontend::gguf::FrontEnd frontend;
+    frontend.add_extension(std::make_shared<ov::frontend::DecoderTransformationExtension>(
+        ov::frontend::gguf::pass::MakeStateful()));
+    auto model = frontend.convert(frontend.load(model_path));
+
+    ov::pass::Manager manager;
+    manager.register_pass<ov::frontend::gguf::pass::AdaptToGenAI>();
+    manager.run_passes(model);
+
+    if (enable_save_ov_model) {
+        std::filesystem::path gguf_model_path(model_path);
+        std::filesystem::path save_path = gguf_model_path.parent_path() / "openvino_model.xml";
+        ov::genai::utils::save_openvino_model(model, save_path.string(), true);
+    }
+
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
-        std::chrono::high_resolution_clock::now() - load_finish_time).count();
+        std::chrono::high_resolution_clock::now() - start_time).count();
     ss.str("");
-    ss << "Model generation done. Time: " << duration << "ms";
+    ss << "GGUF model conversion done. Time: " << duration << "ms";
     ov::genai::utils::print_gguf_debug_info(ss.str());
 
     return model;