Fall back to legacy pipeline when CB model lacks PagedAttention (e.g. Gemma)

src/inference_executor.hpp

@@ -207,9 +207,17 @@ Status modelInferAsync(ModelInstance& instance, const RequestType* request,
         inferRequest.start_async();
     } catch (std::exception& e) {
         SPDLOG_DEBUG("caught exception in ov::InferRequest.start_async: {}", e.what());
+        // start_async threw, so the completion callback will never fire to release the
+        // captured stream-id guard; clear the callback to destroy it and return the
+        // infer-request slot to the pool (otherwise the slot leaks -> #2871).
+        inferRequest.set_callback([](std::exception_ptr) {});
         return StatusCode::OV_INTERNAL_INFERENCE_ERROR;
     } catch (...) {
         SPDLOG_DEBUG("caught exception in ov::InferRequest.start_async");
+        // start_async threw, so the completion callback will never fire to release the
+        // captured stream-id guard; clear the callback to destroy it and return the
+        // infer-request slot to the pool (otherwise the slot leaks -> #2871).
+        inferRequest.set_callback([](std::exception_ptr) {});
         return StatusCode::OV_INTERNAL_INFERENCE_ERROR;
     }
     return StatusCode::OK;

src/llm/language_model/continuous_batching/servable_initializer.cpp

@@ -253,7 +253,16 @@ Status ContinuousBatchingServableInitializer::initialize(std::shared_ptr<GenAiSe
             properties->pluginConfig, properties->tokenizerPluginConfig);
         properties->tokenizer = properties->pipeline->get_tokenizer();
     } catch (const std::exception& e) {
-        SPDLOG_ERROR("Error during llm node initialization for models_path: {} exception: {}", parsedModelsPath, e.what());
+        // Some architectures (e.g. Gemma) export without a ScaledDotProductAttention op, so the
+        // SDPAToPagedAttention transformation required by continuous batching cannot be applied.
+        // Report this distinctly (at warning level) so callers can fall back to the legacy pipeline
+        // when appropriate, rather than logging it as a hard error on an otherwise-recoverable path.
+        const std::string error = e.what();
+        if (error.find("ScaledDotProductAttention") != std::string::npos) {
+            SPDLOG_WARN("Model at models_path: {} does not support PagedAttention required by continuous batching: {}", parsedModelsPath, error);
+            return StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED;
+        }
+        SPDLOG_ERROR("Error during llm node initialization for models_path: {} exception: {}", parsedModelsPath, error);
         return StatusCode::LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED;
     } catch (...) {
         SPDLOG_ERROR("Error during llm node initialization for models_path: {}", parsedModelsPath);

src/llm/servable_initializer.cpp

@@ -430,11 +430,21 @@ Status initializeGenAiServable(std::shared_ptr<GenAiServable>& servable, const :
         if (status != StatusCode::OK) {
             return status;
         }
+        // When the pipeline type was auto-selected (not explicitly requested by the user), continuous
+        // batching can gracefully fall back to the legacy pipeline for models that do not support
+        // PagedAttention (e.g. Gemma, exported without a ScaledDotProductAttention op). When the user
+        // explicitly requested *_CB we surface the error instead of silently changing their request.
+        const bool autoSelectedPipelineType = (nodeOptions.pipeline_type() == mediapipe::LLMCalculatorOptions::AUTO);
         if (pipelineType == PipelineType::LM_CB) {
             SPDLOG_LOGGER_INFO(modelmanager_logger, "Initializing Language Model Continuous Batching servable");
             ContinuousBatchingServableInitializer cbServableInitializer;
             servable = std::make_shared<ContinuousBatchingServable>();
             status = cbServableInitializer.initialize(servable, nodeOptions, graphPath);
+            if (status == StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED && autoSelectedPipelineType) {
+                SPDLOG_LOGGER_WARN(modelmanager_logger, "Model does not support PagedAttention, falling back to Language Model Legacy servable");
+                LegacyServableInitializer legacyServableInitializer;
+                status = legacyServableInitializer.initialize(servable, nodeOptions, graphPath);
+            }
             if (status != StatusCode::OK) {
                 SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during LLM node resources initialization: {}", status.string());
                 return status;
@@ -446,6 +456,11 @@ Status initializeGenAiServable(std::shared_ptr<GenAiServable>& servable, const :
             ContinuousBatchingServableInitializer cbServableInitializer;
             servable = std::make_shared<VisualLanguageModelServable>();
             status = cbServableInitializer.initialize(servable, nodeOptions, graphPath);
+            if (status == StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED && autoSelectedPipelineType) {
+                SPDLOG_LOGGER_WARN(modelmanager_logger, "Model does not support PagedAttention, falling back to Visual Language Model Legacy servable");
+                VisualLanguageModelLegacyServableInitializer legacyServableInitializer;
+                status = legacyServableInitializer.initialize(servable, nodeOptions, graphPath);
+            }
             if (status != StatusCode::OK) {
                 SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during LLM node resources initialization: {}", status.string());
                 return status;

src/status.cpp

@@ -226,6 +226,7 @@ const std::unordered_map<StatusCode, std::string> Status::statusMessageMap = {
     {StatusCode::LLM_NODE_DIRECTORY_DOES_NOT_EXIST, "The LLM Node workspace path does not exist"},
     {StatusCode::LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE, "The LLM Node workspace path does not exist and not gguf model file"},
     {StatusCode::LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED, "The LLM Node resource initialization failed"},
+    {StatusCode::LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED, "The model does not support PagedAttention required by the continuous batching pipeline"},
     {StatusCode::LLM_NODE_MISSING_OPTIONS, "The LLM Node is missing options definition"},
     {StatusCode::LLM_NODE_MISSING_NAME, "The LLM Node is missing name definition"},
 

src/status.hpp

@@ -272,6 +272,7 @@ enum class StatusCode {
     LLM_NODE_DIRECTORY_DOES_NOT_EXIST,
     LLM_NODE_PATH_DOES_NOT_EXIST_AND_NOT_GGUFFILE,
     LLM_NODE_RESOURCE_STATE_INITIALIZATION_FAILED,
+    LLM_NODE_PAGED_ATTENTION_NOT_SUPPORTED,
     LLM_NODE_MISSING_OPTIONS,
     LLM_NODE_MISSING_NAME,