diff --git a/src/xagent/core/model/chat/basic/router.py b/src/xagent/core/model/chat/basic/router.py index bd8704f0ab..568fc72df8 100644 --- a/src/xagent/core/model/chat/basic/router.py +++ b/src/xagent/core/model/chat/basic/router.py @@ -656,10 +656,11 @@ def __init__( self.context_window = context_window ability_source = downstream if router.uses_configured_candidates else router abilities = list(getattr(ability_source, "abilities", router.abilities)) - for modality in input_modalities: - ability = _MODALITY_ABILITIES.get(modality) - if ability is not None and ability not in abilities: - abilities.append(ability) + if not router.uses_configured_candidates: + for modality in input_modalities: + ability = _MODALITY_ABILITIES.get(modality) + if ability is not None and ability not in abilities: + abilities.append(ability) self._abilities = abilities @property diff --git a/src/xagent/web/api/a2a.py b/src/xagent/web/api/a2a.py index 40f2a193d3..ac2dd57c16 100644 --- a/src/xagent/web/api/a2a.py +++ b/src/xagent/web/api/a2a.py @@ -43,11 +43,13 @@ task_state, task_to_a2a, ) +from ..services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE from ..services.db_runtime import ( cancel_and_drain_async_task, drain_async_task_cancellation_safe, run_db_io_cancellation_safe, ) +from ..services.llm_utils import AutoModelUnavailableError from ..services.task_command_transport import ( COMMAND_COMPLETED, COMMAND_FAILED, @@ -614,10 +616,17 @@ async def inject_user_message() -> tuple[Any, UserMessageInjectionOutcome]: status_code=503, details={"taskId": task_id}, ) from exc - except BaseException: + except BaseException as exc: if not ownership_transferred and not prelease_cleanup_done: cleanup_task = asyncio.create_task(stop_and_restore_prelease()) await drain_async_task_cancellation_safe(cleanup_task) + if isinstance(exc, AutoModelUnavailableError): + raise a2a_error( + "unsupported_operation", + CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE, + status_code=409, + details={"code": "auto_model_unavailable"}, + ) from exc raise finally: if not ownership_transferred and not prelease_cleanup_done: diff --git a/src/xagent/web/api/admin_users.py b/src/xagent/web/api/admin_users.py index 381b4c7a65..3a207dc3fb 100644 --- a/src/xagent/web/api/admin_users.py +++ b/src/xagent/web/api/admin_users.py @@ -213,9 +213,11 @@ def _record_settled_bindings_sync( def _delete_user_rows_sync(*, user_id: int) -> bool: """Delete one user and every row it owns in an operation-local session.""" - from ..models.auto_model import AutoModelConfig + from ..models.auto_model import AutoModelCandidate, AutoModelConfig from ..models.mcp import UserMCPServer from ..models.model import Model as DBModel + from ..models.user import UserModel + from ..services.model_service import _is_model_visible_to_user session_factory = get_session_local() delete_db = session_factory() @@ -245,8 +247,34 @@ def _delete_user_rows_sync(*, user_id: int) -> bool: synchronize_session=False ) - # Delete the user (UserModel and UserDefaultModel have cascade delete) + affected_candidates = ( + delete_db.query(AutoModelCandidate) + .join(AutoModelConfig) + .filter( + AutoModelCandidate.target_model_id.in_( + select(UserModel.model_id).where(UserModel.user_id == user_id) + ), + AutoModelConfig.user_id != user_id, + ) + .all() + ) + + # Delete the user, then evaluate the surviving grants. Other owners or + # shared grants can still make a target available to a bound Auto. delete_db.delete(user) + delete_db.flush() + changed_configs = set() + for candidate in affected_candidates: + config = candidate.config + if _is_model_visible_to_user( + delete_db, candidate.target_model_id, int(config.user_id) + ): + continue + if config.fallback_model_id == candidate.target_model_id: + config.fallback_model_id = None + changed_configs.add(int(config.id)) + delete_db.delete(candidate) + ModelStore(delete_db).refresh_auto_model_abilities(list(changed_configs)) delete_db.commit() ModelStore(delete_db).invalidate_after_user_delete() return True diff --git a/src/xagent/web/api/agents.py b/src/xagent/web/api/agents.py index 2178e9990b..a14f1959f4 100644 --- a/src/xagent/web/api/agents.py +++ b/src/xagent/web/api/agents.py @@ -63,7 +63,8 @@ owned_agent_clause, ) from ..services.api_keys import AgentApiKeyService, KeyRotationConflict -from ..services.llm_utils import UserAwareModelStorage +from ..services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE +from ..services.llm_utils import AutoModelUnavailableError, UserAwareModelStorage from ..services.workforce_access import get_visible_agent_ids from ..tools.config import WebToolConfig from ..user_isolated_memory import UserContext @@ -592,6 +593,8 @@ async def optimize_instructions( return {"optimized_instructions": content} + except AutoModelUnavailableError as exc: + raise HTTPException(409, detail=CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE) from exc except Exception as e: logger.error(f"Failed to optimize instructions: {e}") raise HTTPException(status_code=500, detail=str(e)) diff --git a/src/xagent/web/api/model.py b/src/xagent/web/api/model.py index 9433ef3448..ffdfaf83af 100644 --- a/src/xagent/web/api/model.py +++ b/src/xagent/web/api/model.py @@ -2032,8 +2032,14 @@ async def update_model( effective_model_name = update_data.get("model_name", db_model.model_name) _validate_provider_model_name(effective_provider, effective_model_name) effective_category = update_data.get("category", db_model.category) - incompatible_with_auto = effective_category != "llm" or is_auto_router_model( - effective_provider, effective_model_name + identity_changed = any( + field in update_data and update_data[field] != getattr(db_model, field) + for field in ("model_provider", "model_name", "base_url") + ) + incompatible_with_auto = ( + identity_changed + or effective_category != "llm" + or is_auto_router_model(effective_provider, effective_model_name) ) if incompatible_with_auto: own_auto_reference = ( @@ -2051,13 +2057,37 @@ async def update_model( if own_auto_reference is not None: raise HTTPException( 409, - detail="Cannot change this model into a non-LLM or Auto model while an Auto configuration uses it.", + detail="Remove this model from your Auto configuration before changing its identity or category.", ) model_store.prune_external_auto_references( model_id=int(db_model.id), owner_user_id=int(user.id), ) + auto_candidates = ( + db.query(AutoModelCandidate) + .filter(AutoModelCandidate.target_model_id == db_model.id) + .all() + ) + if auto_candidates and "abilities" in update_data: + from ..services.auto_model_service import ( + load_router_profile_catalog, + validate_candidate_modalities, + ) + + try: + catalog = load_router_profile_catalog() + for candidate in auto_candidates: + validate_candidate_modalities( + catalog, + str(candidate.routing_model_id), + update_data["abilities"] or [], + ) + except AutoModelConfigurationError as exc: + raise HTTPException(409, detail=str(exc)) from exc + except AutoModelDependencyError as exc: + raise HTTPException(503, detail=str(exc)) from exc + for field, value in update_data.items(): # Don't update api_key with empty string if field == "api_key" and value == "": @@ -2069,6 +2099,11 @@ async def update_model( if hasattr(db_model, field): setattr(db_model, field, value) + if auto_candidates: + model_store.refresh_auto_model_abilities( + [int(candidate.config_id) for candidate in auto_candidates] + ) + if share_with_users is not None: try: model_store.set_model_sharing( diff --git a/src/xagent/web/api/v1/errors.py b/src/xagent/web/api/v1/errors.py index 75f7fd270f..5f6b5e122b 100644 --- a/src/xagent/web/api/v1/errors.py +++ b/src/xagent/web/api/v1/errors.py @@ -27,6 +27,8 @@ from fastapi import Request from fastapi.responses import JSONResponse +from ...services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + class V1ErrorCode(str, Enum): """Stable error codes for ``/v1/*`` responses. @@ -110,6 +112,7 @@ class V1ErrorCode(str, Enum): # Server-side bug. Detail is sanitized; the raw exception stays in # the server log. INTERNAL_ERROR = "internal_error" + AUTO_MODEL_UNAVAILABLE = "auto_model_unavailable" CONNECTOR_NOT_FOUND = "connector_not_found" INVALID_RUNTIME_CONTEXT = "invalid_runtime_context" @@ -178,6 +181,7 @@ class V1ErrorCode(str, Enum): "Monthly execution quota exceeded for this client application." ), V1ErrorCode.INTERNAL_ERROR: "Internal server error.", + V1ErrorCode.AUTO_MODEL_UNAVAILABLE: CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE, V1ErrorCode.CONNECTOR_NOT_FOUND: "Connector not found or not accessible.", V1ErrorCode.INVALID_RUNTIME_CONTEXT: "Invalid connector runtime context.", V1ErrorCode.MISSING_RUNTIME_CONTEXT: "Required connector runtime context is missing.", diff --git a/src/xagent/web/api/v1/task_reply.py b/src/xagent/web/api/v1/task_reply.py index 0c9734c7fe..8be144d3b4 100644 --- a/src/xagent/web/api/v1/task_reply.py +++ b/src/xagent/web/api/v1/task_reply.py @@ -58,6 +58,7 @@ drain_async_task_cancellation_safe, run_db_io_cancellation_safe, ) +from ...services.llm_utils import AutoModelUnavailableError from ...services.task_execution_controller import TaskControlState from ...services.task_interaction_close import ( active_interaction_id_sync, @@ -599,10 +600,12 @@ async def inject_user_message() -> tuple[Any, bool]: # failure, so an unrecognized failure mode never silently # collapses into a data-losing branch. raise V1ApiError(V1ErrorCode.TEMPORARILY_UNAVAILABLE, 503) from exc - except BaseException: + except BaseException as exc: if not ownership_transferred and not prelease_cleanup_done: cleanup_task = asyncio.create_task(stop_and_restore_prelease()) await drain_async_task_cancellation_safe(cleanup_task) + if isinstance(exc, AutoModelUnavailableError): + raise V1ApiError(V1ErrorCode.AUTO_MODEL_UNAVAILABLE, 409) from exc raise finally: if not ownership_transferred and not prelease_cleanup_done: diff --git a/src/xagent/web/api/websocket.py b/src/xagent/web/api/websocket.py index 67d5caccfb..abc7bfa125 100644 --- a/src/xagent/web/api/websocket.py +++ b/src/xagent/web/api/websocket.py @@ -354,7 +354,7 @@ def create_terminal_task_error_event( failure this path exists to remove. A bad optional argument costs that argument and nothing else. The rejection is logged with its stack. - ``code`` must be a member of ``CONNECTOR_RUNTIME_CLIENT_ERROR_CODES``. + ``code`` must be a connector-runtime code or ``AUTO_MODEL_UNAVAILABLE``. """ # Python annotations are not enforced at run time, so the mypy gate on the @@ -370,7 +370,11 @@ def create_terminal_task_error_event( # and an unhashable value would raise inside the membership test on a # path whose whole point is that it never raises. if code is not None and ( - not isinstance(code, str) or code not in CONNECTOR_RUNTIME_CLIENT_ERROR_CODES + not isinstance(code, str) + or ( + code not in CONNECTOR_RUNTIME_CLIENT_ERROR_CODES + and code != ClientErrorCode.AUTO_MODEL_UNAVAILABLE.value + ) ): logger.error( "task_id=%s component=terminal-error-frame dropped=code " @@ -473,6 +477,8 @@ def client_safe_error_message( Read a passing sweep as "the recognized egress shapes are clean", never as "arbitrary Python data flow cannot reach a client raw". """ + if isinstance(error, AutoModelUnavailableError): + return client_error_message(ClientErrorCode.AUTO_MODEL_UNAVAILABLE) if not isinstance(error, ClientVisibleError): return fallback message = str(error) @@ -10514,8 +10520,17 @@ async def send_builder_outbound_message(payload: Dict[str, Any]) -> None: except Exception as e: logger.error("Error handling builder chat: %s", e, exc_info=True) + error_metadata = {} + if isinstance(e, AutoModelUnavailableError): + error_metadata["error_code"] = ClientErrorCode.AUTO_MODEL_UNAVAILABLE.value await websocket.send_text( - json.dumps({"type": "error", "message": client_safe_error_message(e)}) + json.dumps( + { + **error_metadata, + "type": "error", + "message": client_safe_error_message(e), + } + ) ) diff --git a/src/xagent/web/api/workforces.py b/src/xagent/web/api/workforces.py index 818c56e673..c3ce1b324e 100644 --- a/src/xagent/web/api/workforces.py +++ b/src/xagent/web/api/workforces.py @@ -33,12 +33,14 @@ get_agent_team_scope, owns_agent, ) +from ..services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE from ..services.deployments import ( get_deployment, get_or_create_deployment, new_share_token, new_widget_key, ) +from ..services.llm_utils import AutoModelUnavailableError from ..services.trace_event_types import GENERAL_ERROR_EVENT_TYPES from ..services.trace_message_storage import decode_trace_events_data from ..services.triggers import unregister_deleted_trigger_bindings @@ -622,7 +624,10 @@ async def create_workforce_from_prompt_endpoint( db: Session = Depends(get_db), user: User = Depends(get_current_user), ) -> dict[str, Any]: - result = await create_workforce_from_prompt(db, user, prompt=request.prompt) + try: + result = await create_workforce_from_prompt(db, user, prompt=request.prompt) + except AutoModelUnavailableError as exc: + raise HTTPException(409, detail=CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE) from exc workforce = _reload_workforce(db, result.workforce) return _serialize_workforce_detail( workforce, user, get_agent_team_scope(db, int(user.id)) diff --git a/src/xagent/web/channels/feishu/bot.py b/src/xagent/web/channels/feishu/bot.py index 2c2288f8e1..803eccf0bc 100644 --- a/src/xagent/web/channels/feishu/bot.py +++ b/src/xagent/web/channels/feishu/bot.py @@ -27,6 +27,7 @@ register_channel_uploaded_files, update_channel_task_fields, ) +from ...services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE from ...services.db_runtime import ( cancel_and_drain_async_task, drain_async_task_cancellation_safe, @@ -34,6 +35,7 @@ ) from ...services.execution_result_projection import project_execution_result_for_channel from ...services.file_turn import normalize_attachments_for_persistence +from ...services.llm_utils import AutoModelUnavailableError from ...services.managed_task_lease import ManagedTaskLease from ...services.task_execution_context_service import ( materialize_task_execution_recovery_state, @@ -429,7 +431,10 @@ async def _process_messages_batch( ) return await self._send_text( - chat_id, "Sorry, an error occurred while processing your request." + chat_id, + CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + if isinstance(e, AutoModelUnavailableError) + else "Sorry, an error occurred while processing your request.", ) finally: if managed_lease is not None: diff --git a/src/xagent/web/channels/slack/bot.py b/src/xagent/web/channels/slack/bot.py index 3c227ba0e1..2f0ef88dac 100644 --- a/src/xagent/web/channels/slack/bot.py +++ b/src/xagent/web/channels/slack/bot.py @@ -36,6 +36,7 @@ register_channel_uploaded_files, update_channel_task_fields, ) +from ...services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE from ...services.db_runtime import ( cancel_and_drain_async_task, drain_async_task_cancellation_safe, @@ -47,6 +48,7 @@ build_uploaded_files_context, normalize_attachments_for_persistence, ) +from ...services.llm_utils import AutoModelUnavailableError from ...services.managed_task_lease import ManagedTaskLease from ...services.task_execution_context_service import ( materialize_task_execution_recovery_state, @@ -672,7 +674,9 @@ async def _process_event( claimed_task_id, ) return - if isinstance(error, SlackFileDownloadError): + if isinstance(error, AutoModelUnavailableError): + error_text = CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + elif isinstance(error, SlackFileDownloadError): error_text = ( "I couldn't download the attached Slack file(s). " "Please try uploading them again." diff --git a/src/xagent/web/services/auto_model_service.py b/src/xagent/web/services/auto_model_service.py index 54f1923194..70972cf477 100644 --- a/src/xagent/web/services/auto_model_service.py +++ b/src/xagent/web/services/auto_model_service.py @@ -64,6 +64,27 @@ def list_router_profiles() -> list[dict[str, Any]]: ] +def validate_candidate_modalities( + catalog: Any, profile_id: str, abilities: Iterable[str] +) -> None: + """A profile must describe the saved endpoint's declared input capabilities.""" + if profile_id not in catalog.known_model_ids(): + raise AutoModelConfigurationError(f"Unknown Auto profile {profile_id!r}") + profile = catalog.get(profile_id) + modality_abilities = {"image": "vision", "audio": "audio", "video": "video"} + profile_modalities = set(profile.input_modalities) & modality_abilities.keys() + target_modalities = { + modality + for modality, ability in modality_abilities.items() + if ability in abilities + } + if profile_modalities != target_modalities: + raise AutoModelConfigurationError( + f"Profile {profile_id!r} input modalities do not match the candidate model's abilities. " + "Choose a matching profile or correct the model's abilities." + ) + + class AutoModelService: def __init__(self, db: Session) -> None: self.db = db @@ -104,6 +125,13 @@ def upsert_config( + ", ".join(str(model_id) for model_id in missing_target_ids) ) + for candidate in request.candidates: + validate_candidate_modalities( + catalog, + candidate.routing_model_id, + targets[candidate.target_model_id].abilities or [], + ) + config = ( self.db.query(AutoModelConfig) .filter(AutoModelConfig.user_id == user_id) diff --git a/src/xagent/web/services/builder_chat_runtime.py b/src/xagent/web/services/builder_chat_runtime.py index f299b19918..f075ca701e 100644 --- a/src/xagent/web/services/builder_chat_runtime.py +++ b/src/xagent/web/services/builder_chat_runtime.py @@ -64,7 +64,18 @@ def _load_builder_chat_runtime_inputs_sync( if not llm or compact_llm is None: default_llm, _fast_llm, _vision_llm, default_compact_llm = ( - resolver.get_configured_defaults(user_id=user_id) + resolver.get_configured_defaults( + user_id=user_id, + config_types=tuple( + kind + for kind, missing in ( + ("general", llm is None), + ("compact", compact_llm is None), + ) + if missing + ), + fallback_llm=llm, + ) ) if not llm: llm = default_llm diff --git a/src/xagent/web/services/llm_utils.py b/src/xagent/web/services/llm_utils.py index ba3ca8421e..35aa7e4c53 100644 --- a/src/xagent/web/services/llm_utils.py +++ b/src/xagent/web/services/llm_utils.py @@ -750,7 +750,13 @@ def _build_configured_router_resolver( ) -> tuple[ChatModelConfig, Callable[[str], BaseLLM]]: """Snapshot the user's Auto bindings into a resolver for this run.""" - from .auto_model_service import AUTO_ROUTER_CONFIG_NAME + from .auto_model_service import ( + AUTO_ROUTER_CONFIG_NAME, + AutoModelConfigurationError, + AutoModelDependencyError, + load_router_profile_catalog, + validate_candidate_modalities, + ) from .model_service import _is_model_visible_to_user config = ( @@ -800,6 +806,15 @@ def _build_configured_router_resolver( "Auto model has no active configured candidates" ) + try: + catalog = load_router_profile_catalog() + for profile_id, target_cfg in targets_by_profile.items(): + validate_candidate_modalities( + catalog, profile_id, target_cfg.abilities or [] + ) + except (AutoModelConfigurationError, AutoModelDependencyError) as exc: + raise AutoModelUnavailableError(str(exc)) from exc + profile_ids = list(targets_by_profile) configured = router_cfg.model_copy( update={ @@ -839,7 +854,11 @@ def _create_default_model( return self.core_storage.create_llm_instance(model_config) def get_configured_defaults( - self, user_id: Optional[int] = None + self, + user_id: Optional[int] = None, + *, + config_types: tuple[str, ...] = ("general", "small_fast", "visual", "compact"), + fallback_llm: Optional[BaseLLM] = None, ) -> Tuple[ Optional[BaseLLM], Optional[BaseLLM], Optional[BaseLLM], Optional[BaseLLM] ]: @@ -848,6 +867,8 @@ def get_configured_defaults( Args: user_id: User ID for multi-tenant model resolution. If None, uses admin defaults. + config_types: Only these default slots are instantiated. + fallback_llm: Explicit general LLM to use when a requested specialized slot is unset. Returns: Tuple of (default_llm, fast_llm, vision_llm, compact_llm) @@ -877,7 +898,11 @@ def get_configured_defaults( .first() ) - if general_default and general_default.model: + if ( + "general" in config_types + and general_default + and general_default.model + ): from .model_service import _is_model_visible_to_user if _is_model_visible_to_user( @@ -902,7 +927,7 @@ def get_configured_defaults( .first() ) - if fast_default and fast_default.model: + if "small_fast" in config_types and fast_default and fast_default.model: from .model_service import _is_model_visible_to_user if _is_model_visible_to_user( @@ -927,7 +952,7 @@ def get_configured_defaults( .first() ) - if vision_default and vision_default.model: + if "visual" in config_types and vision_default and vision_default.model: from .model_service import _is_model_visible_to_user if _is_model_visible_to_user( @@ -952,7 +977,11 @@ def get_configured_defaults( .first() ) - if compact_default and compact_default.model: + if ( + "compact" in config_types + and compact_default + and compact_default.model + ): from .model_service import _is_model_visible_to_user if _is_model_visible_to_user( @@ -975,9 +1004,7 @@ def get_configured_defaults( UserDefaultModel.model_id == UserModel.model_id, ) .filter( - UserDefaultModel.config_type.in_( - ["general", "small_fast", "visual", "compact"] - ), + UserDefaultModel.config_type.in_(config_types), UserModel.is_shared.is_(True), UserDefaultModel.user_id.in_(visible_ids), ) @@ -1002,19 +1029,28 @@ def get_configured_defaults( admin_default.model, user_id ) - # Fallback to environment variables if no configured models - if not default_llm: - default_llm = create_llm_from_env() - if default_llm: - logger.info("Using environment variables for default LLM") + requested_defaults = { + "general": default_llm, + "small_fast": fast_llm, + "visual": vision_llm, + "compact": compact_llm, + } + if any(requested_defaults[kind] is None for kind in config_types): + default_llm = default_llm or fallback_llm + if default_llm is None and "general" not in config_types: + default_llm, _, _, _ = self.get_configured_defaults( + user_id, config_types=("general",) + ) + if default_llm is None: + default_llm = create_llm_from_env() - if not fast_llm: + if "small_fast" in config_types and not fast_llm: fast_llm = default_llm - if not vision_llm: + if "visual" in config_types and not vision_llm: vision_llm = default_llm - if not compact_llm: + if "compact" in config_types and not compact_llm: compact_llm = default_llm return default_llm, fast_llm, vision_llm, compact_llm @@ -1074,7 +1110,9 @@ def resolve_llms_from_names( logger.warning( f"Default LLM '{default_name}' not found or no access, falling back to configured default" ) - default_llm, _, _, _ = self.get_configured_defaults(user_id) + default_llm, _, _, _ = self.get_configured_defaults( + user_id, config_types=("general",) + ) # Get fast LLM (optional) fast_llm = None @@ -1095,14 +1133,20 @@ def resolve_llms_from_names( default_fast_llm = None default_vision_llm = None default_compact_llm = None - needs_specialized_defaults = ( - (bool(fast_name) and fast_llm is None) - or (bool(vision_name) and vision_llm is None) - or compact_llm is None + missing_defaults = tuple( + kind + for kind, needed in ( + ("small_fast", bool(fast_name) and fast_llm is None), + ("visual", bool(vision_name) and vision_llm is None), + ("compact", compact_llm is None), + ) + if needed ) - if needs_specialized_defaults: + if missing_defaults: _, default_fast_llm, default_vision_llm, default_compact_llm = ( - self.get_configured_defaults(user_id) + self.get_configured_defaults( + user_id, config_types=missing_defaults, fallback_llm=default_llm + ) ) if fast_name and not fast_llm: diff --git a/src/xagent/web/services/model_service.py b/src/xagent/web/services/model_service.py index 76960306df..330751edd5 100644 --- a/src/xagent/web/services/model_service.py +++ b/src/xagent/web/services/model_service.py @@ -21,6 +21,7 @@ from ...core.model.image.openai import OpenAIImageModel from ...core.model.image.xinference import XinferenceImageModel from ...core.model.video.base import BaseVideoModel +from .llm_utils import AutoModelUnavailableError logger = logging.getLogger(__name__) @@ -212,6 +213,8 @@ def get_default_vision_model( model_db, admin_vision_defaults[0].model, user_id ) + except AutoModelUnavailableError: + raise except Exception as e: logger.warning(f"Failed to get vision model from database: {e}") pass @@ -278,6 +281,8 @@ def get_default_model(user_id: Optional[int] = None) -> Optional[BaseLLM]: db, admin_defaults[0].model, user_id ) + except AutoModelUnavailableError: + raise except Exception as e: logger.warning(f"Failed to get default model from database: {e}") pass @@ -344,6 +349,8 @@ def get_fast_model(user_id: Optional[int] = None) -> Optional[BaseLLM]: db, admin_fast_defaults[0].model, user_id ) + except AutoModelUnavailableError: + raise except Exception as e: logger.warning(f"Failed to get fast model from database: {e}") pass @@ -410,6 +417,8 @@ def get_compact_model(user_id: Optional[int] = None) -> Optional[BaseLLM]: db, admin_compact_defaults[0].model, user_id ) + except AutoModelUnavailableError: + raise except Exception as e: logger.warning(f"Failed to get compact model from database: {e}") pass diff --git a/src/xagent/web/services/model_store.py b/src/xagent/web/services/model_store.py index ce4f6fc199..9e5655389c 100644 --- a/src/xagent/web/services/model_store.py +++ b/src/xagent/web/services/model_store.py @@ -469,6 +469,29 @@ def delete_user_default_model( invalidate_model_cache(None if default_was_shared else user_id) return user_default + def refresh_auto_model_abilities(self, config_ids: list[int]) -> None: + if not config_ids: + return + + from ..models.auto_model import AutoModelCandidate, AutoModelConfig + from .auto_model_service import AutoModelService + + self.db.flush() + for config in self.db.query(AutoModelConfig).filter( + AutoModelConfig.id.in_(config_ids) + ): + targets = ( + self.db.query(DBModel) + .join( + AutoModelCandidate, AutoModelCandidate.target_model_id == DBModel.id + ) + .filter(AutoModelCandidate.config_id == config.id) + .all() + ) + AutoModelService._update_router_model_abilities( + config.router_model, targets + ) + def commit_model_update( self, *, user_id: int, db_model: DBModel, invalidate_globally: bool ) -> None: @@ -551,7 +574,7 @@ def prune_external_auto_references( AutoModelConfig.id.in_(external_config_ids), AutoModelConfig.fallback_model_id == model_id, ).update({AutoModelConfig.fallback_model_id: None}, synchronize_session=False) - return int( + deleted = int( self.db.query(AutoModelCandidate) .filter( AutoModelCandidate.config_id.in_(external_config_ids), @@ -559,6 +582,8 @@ def prune_external_auto_references( ) .delete(synchronize_session=False) ) + self.refresh_auto_model_abilities(external_config_ids) + return deleted def delete_model( self, *, model_storage: CoreStorage, user_model: UserModel diff --git a/src/xagent/web/services/task_orchestrator.py b/src/xagent/web/services/task_orchestrator.py index e133ead477..9f7073ecfa 100644 --- a/src/xagent/web/services/task_orchestrator.py +++ b/src/xagent/web/services/task_orchestrator.py @@ -74,6 +74,8 @@ ) from .client_error_messages import ( CLIENT_SAFE_TASK_FAILURE, + ClientErrorCode, + client_error_message, connector_runtime_client_code, connector_runtime_client_message, required_mcp_unavailable_client_message, @@ -89,6 +91,7 @@ ) from .file_turn import bind_turn_files_no_commit from .hot_path_cache import invalidate_task_cache +from .llm_utils import AutoModelUnavailableError from .mcp_runtime import ( MCPBuiltinOAuthActorPolicy, MCPBuiltinOAuthActorPolicyRequiredError, @@ -1965,6 +1968,15 @@ async def execute_owned_run() -> None: fallback=CLIENT_SAFE_TASK_FAILURE, ) ) + elif isinstance(setup_or_run_err, AutoModelUnavailableError): + broadcast_error_code = ( + ClientErrorCode.AUTO_MODEL_UNAVAILABLE.value + ) + broadcast_error_message = client_error_message( + ClientErrorCode.AUTO_MODEL_UNAVAILABLE + ) + settlement_error = broadcast_error_message + client_history_message_type = CLIENT_SAFE_FAILURE_MESSAGE_TYPE elif isinstance(setup_or_run_err, ConnectorRuntimeError): # This exception's message is a curated public-safe # sentence -- it says a runtime input is missing, not diff --git a/tests/web/api/test_a2a_api.py b/tests/web/api/test_a2a_api.py index 1826172637..b247a9c321 100644 --- a/tests/web/api/test_a2a_api.py +++ b/tests/web/api/test_a2a_api.py @@ -37,6 +37,8 @@ A2AApiError, A2ATaskSnapshot, ) +from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE +from xagent.web.services.llm_utils import AutoModelUnavailableError from xagent.web.services.task_command_transport import ( COMMAND_FAILED, MAX_COMMAND_FAILURES, @@ -1726,6 +1728,7 @@ def _resume_error_task(agent_id: int, *, context_id: str) -> int: @pytest.mark.parametrize( ("error", "expected_status"), [ + (AutoModelUnavailableError("private model details"), 409), (CheckpointUnavailableError("checkpoint query failed"), 503), (CheckpointCorruptError("all matching rows undecodable"), 400), ( @@ -1768,6 +1771,12 @@ def test_checkpoint_read_error_maps_to_distinct_status_and_restores_waiting( ) assert response.status_code == expected_status, response.text + if isinstance(error, AutoModelUnavailableError): + assert response.json()["error"]["message"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + assert ( + response.json()["error"]["details"][0]["metadata"]["code"] + == "auto_model_unavailable" + ) db = _direct_db_session() try: recovered = db.query(Task).filter(Task.id == task_id).one() diff --git a/tests/web/api/test_websocket_builder_chat.py b/tests/web/api/test_websocket_builder_chat.py index 0e26e5ba9b..8b7b363aa3 100644 --- a/tests/web/api/test_websocket_builder_chat.py +++ b/tests/web/api/test_websocket_builder_chat.py @@ -15,6 +15,8 @@ ) from xagent.web.models.user import User from xagent.web.services.builder_chat_runtime import BuilderChatRuntimeInputs +from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE +from xagent.web.services.llm_utils import AutoModelUnavailableError @pytest.mark.asyncio @@ -533,5 +535,22 @@ async def test_handle_builder_chat_no_llm() -> None: assert "No LLM configured" in sent_data["message"] +@pytest.mark.asyncio +async def test_builder_auto_unavailable_has_safe_message_and_code(): + websocket = AsyncMock() + user = SimpleNamespace(id=1, is_admin=False) + with patch( + "xagent.web.services.builder_chat_runtime.load_builder_chat_runtime_inputs", + AsyncMock(side_effect=AutoModelUnavailableError("private model details")), + ): + await handle_builder_chat(websocket, {"message": "Create an agent"}, user) + payload = json.loads(websocket.send_text.call_args.args[0]) + assert payload == { + "type": "error", + "message": CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE, + "error_code": "auto_model_unavailable", + } + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/web/api/v1/test_task_reply.py b/tests/web/api/v1/test_task_reply.py index 2a59721c81..46b8126366 100644 --- a/tests/web/api/v1/test_task_reply.py +++ b/tests/web/api/v1/test_task_reply.py @@ -27,6 +27,8 @@ from xagent.web.models.task import Task, TaskStatus, TraceEvent from xagent.web.models.task_interaction import TaskInteractionRequest from xagent.web.schemas.v1 import ReplyRequest +from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE +from xagent.web.services.llm_utils import AutoModelUnavailableError from xagent.web.services.task_execution_controller import TaskControlState from xagent.web.services.task_lease_service import TaskLease, current_task_lease @@ -794,6 +796,11 @@ def test_reply_checkpoint_missing_restore_clears_an_unpaired_marker(mock_start_t @pytest.mark.parametrize( ("error", "expected_status", "expected_code"), [ + ( + AutoModelUnavailableError("private model details"), + 409, + "auto_model_unavailable", + ), ( CheckpointCorruptError("all matching rows undecodable"), 409, @@ -843,6 +850,8 @@ def test_reply_checkpoint_read_error_maps_to_distinct_code( assert resp.status_code == expected_status, resp.text assert resp.json()["error"]["code"] == expected_code + if isinstance(error, AutoModelUnavailableError): + assert resp.json()["error"]["message"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE db = _direct_db_session() try: diff --git a/tests/web/services/test_auto_model_consistency.py b/tests/web/services/test_auto_model_consistency.py new file mode 100644 index 0000000000..4506b375a4 --- /dev/null +++ b/tests/web/services/test_auto_model_consistency.py @@ -0,0 +1,351 @@ +"""Auto binding, default resolution and lifecycle regressions against a real DB.""" + +import asyncio +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from xagent.core.model.chat.basic.router import RouterLLM, _ResolvedRouterLLM +from xagent.web import models +from xagent.web.api import admin_users, agents +from xagent.web.api import model as model_api +from xagent.web.api import workforces +from xagent.web.auth_dependencies import get_current_user +from xagent.web.models import database +from xagent.web.models.auto_model import AutoModelCandidate, AutoModelConfig +from xagent.web.models.model import Model +from xagent.web.models.user import User, UserDefaultModel, UserModel +from xagent.web.schemas.model import AutoModelConfigUpdate, ModelUpdate +from xagent.web.services import auto_model_service, builder_chat_runtime, model_service +from xagent.web.services.auto_model_service import ( + AutoModelConfigurationError, + AutoModelService, +) +from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE +from xagent.web.services.llm_utils import ( + AutoModelUnavailableError, + UserAwareModelStorage, +) + + +@pytest.fixture +def env(monkeypatch, tmp_path): + monkeypatch.setenv("XAGENT_STORAGE_ROOT", str(tmp_path)) + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + + @event.listens_for(engine, "connect") + def foreign_keys(conn, _): + conn.execute("PRAGMA foreign_keys=ON") + + models.Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + db = factory() + + def get_db(): + yield db + + monkeypatch.setattr(database, "get_db", get_db) + monkeypatch.setattr(admin_users, "get_session_local", lambda: factory) + monkeypatch.setattr(builder_chat_runtime, "get_session_local", lambda: factory) + monkeypatch.setattr(model_service, "_visible_user_ids_hook", None) + profiles = { + "text-a": SimpleNamespace(input_modalities=("text",)), + "text-b": SimpleNamespace(input_modalities=("text",)), + "image-a": SimpleNamespace(input_modalities=("text", "image")), + } + catalog = SimpleNamespace( + known_model_ids=lambda: tuple(profiles), get=profiles.__getitem__ + ) + monkeypatch.setattr( + auto_model_service, "load_router_profile_catalog", lambda: catalog + ) + owner = User(username="owner", password_hash="unused", is_admin=True) + consumer = User(username="consumer", password_hash="unused", is_admin=False) + db.add_all([owner, consumer]) + db.commit() + yield SimpleNamespace(db=db, factory=factory, owner=owner, consumer=consumer) + db.close() + engine.dispose() + + +def target(env, owner, *, name="saved", abilities=None, shared=False): + row = Model( + model_id=name, + category="llm", + model_provider="openai", + model_name="gpt-4", + api_key="fake-test-key", + base_url="https://example.invalid/v1", + abilities=abilities or ["chat"], + is_active=True, + ) + env.db.add(row) + env.db.flush() + env.db.add( + UserModel( + user_id=owner.id, + model_id=row.id, + is_owner=True, + can_edit=True, + can_delete=True, + is_shared=shared, + ) + ) + env.db.commit() + return row + + +def configure(env, user, bindings, *, default=False): + return AutoModelService(env.db).upsert_config( + user_id=user.id, + request=AutoModelConfigUpdate( + candidates=[ + {"routing_model_id": profile, "target_model_id": t.id} + for profile, t in bindings + ], + fallback_model_id=bindings[0][1].id, + set_as_default=default, + ), + ) + + +def broken_default(env, slot="general"): + t = target(env, env.consumer) + cfg = configure(env, env.consumer, [("text-a", t)]) + env.db.add( + UserDefaultModel( + user_id=env.consumer.id, model_id=cfg.router_model_id, config_type=slot + ) + ) + t.is_active = False + env.db.commit() + return cfg + + +@pytest.mark.parametrize( + "profile,abilities", [("image-a", ["chat"]), ("text-a", ["chat", "vision"])] +) +def test_rejects_profile_capability_mismatch_without_saving(env, profile, abilities): + t = target(env, env.consumer, abilities=abilities) + with pytest.raises(AutoModelConfigurationError, match="input modalities"): + configure(env, env.consumer, [(profile, t)]) + assert env.db.query(AutoModelConfig).count() == 0 + + +def test_legacy_mismatched_binding_fails_before_provider_call(env): + t = target(env, env.consumer) + cfg = configure(env, env.consumer, [("text-a", t)]) + cfg.candidates[0].routing_model_id = "image-a" + env.db.commit() + with pytest.raises(AutoModelUnavailableError, match="input modalities"): + UserAwareModelStorage(env.db).get_llm_by_id( + cfg.router_model.model_id, env.consumer.id + ) + + +def test_configured_wrapper_never_adds_profile_only_abilities(): + downstream = SimpleNamespace(abilities=["chat"]) + router = RouterLLM( + candidate_models=["image-a"], downstream_resolver=lambda _: downstream + ) + resolved = _ResolvedRouterLLM( + router=router, + downstream=downstream, + selected_model="image-a", + context_window=None, + input_modalities=("image",), + ) + assert resolved.abilities == ["chat"] + + +@pytest.mark.parametrize( + "changes", + [ + {"model_name": "gpt-4.1"}, + {"model_provider": "deepseek", "model_name": "deepseek-v4-flash"}, + {"base_url": "https://another.invalid/v1"}, + {"abilities": ["chat", "vision"]}, + ], +) +def test_bound_candidate_identity_and_modalities_cannot_change(env, changes): + t = target(env, env.consumer) + configure(env, env.consumer, [("text-a", t)]) + with pytest.raises(HTTPException) as error: + asyncio.run( + model_api.update_model( + t.model_id, ModelUpdate(**changes), db=env.db, user=env.consumer + ) + ) + assert error.value.status_code == 409 + env.db.rollback() + assert t.model_name == "gpt-4" and t.abilities == ["chat"] + + +def test_owner_identity_change_prunes_external_binding(env): + t = target(env, env.owner, shared=True) + cfg = configure(env, env.consumer, [("text-a", t)]) + asyncio.run( + model_api.update_model( + t.model_id, ModelUpdate(model_name="gpt-4.1"), db=env.db, user=env.owner + ) + ) + env.db.expire_all() + assert env.db.query(AutoModelCandidate).filter_by(config_id=cfg.id).count() == 0 + assert env.db.get(AutoModelConfig, cfg.id).fallback_model_id is None + + +@pytest.mark.parametrize("configured_compact", [False, True]) +def test_missing_compact_does_not_resolve_broken_unrelated_general( + env, configured_compact +): + broken_default(env) + healthy = target(env, env.consumer, name="healthy") + if configured_compact: + env.db.add( + UserDefaultModel( + user_id=env.consumer.id, model_id=healthy.id, config_type="compact" + ) + ) + env.db.commit() + resolved = UserAwareModelStorage(env.db).resolve_llms_from_names( + [healthy.model_id] * 3 + [None], env.consumer.id + ) + assert all(llm.model_id == healthy.model_id for llm in resolved) + builder = builder_chat_runtime._load_builder_chat_runtime_inputs_sync( + user_id=env.consumer.id, + requested_file_ids=[], + model_name=healthy.model_id, + compact_model_name=None, + ) + assert builder.llm.model_id == builder.compact_llm.model_id == healthy.model_id + + +@pytest.mark.parametrize( + "slot,getter", + [ + ("general", "get_default_model"), + ("small_fast", "get_fast_model"), + ("visual", "get_default_vision_model"), + ("compact", "get_compact_model"), + ], +) +def test_default_getters_preserve_auto_configuration_error(env, slot, getter): + broken_default(env, slot) + with pytest.raises(AutoModelUnavailableError): + UserAwareModelStorage(env.db).get_configured_defaults(env.consumer.id) + kwargs = {"db": env.db} if slot == "visual" else {} + with pytest.raises(AutoModelUnavailableError): + getattr(model_service, getter)(env.consumer.id, **kwargs) + + +@pytest.mark.parametrize("surface", ["optimize", "workforce"]) +def test_http_entrypoints_map_actual_broken_auto_to_safe_409(env, surface): + broken_default(env) + app = FastAPI() + app.include_router(agents.router if surface == "optimize" else workforces.router) + + def db_override(): + yield env.db + + app.dependency_overrides[agents.get_db] = db_override + app.dependency_overrides[workforces.get_db] = db_override + app.dependency_overrides[get_current_user] = lambda: env.consumer + with TestClient(app) as client: + if surface == "optimize": + response = client.post( + "/api/agents/optimize-instructions", + json={"instructions": "Write a clear summary."}, + ) + else: + response = client.post( + "/api/workforces/from-prompt", json={"prompt": "Write a clear summary."} + ) + assert response.status_code == 409 + assert response.json()["detail"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + + +@pytest.mark.parametrize("retained_grant", [False, True]) +def test_admin_delete_prunes_only_inaccessible_bindings(env, retained_grant): + t = target(env, env.owner, shared=True, abilities=["chat", "tool_calling"]) + cfg = configure(env, env.consumer, [("text-a", t)]) + ids = env.owner.id, env.consumer.id, t.id, cfg.id, cfg.router_model_id + if retained_grant: + env.db.add(UserModel(user_id=env.consumer.id, model_id=t.id, is_owner=True)) + env.db.commit() + env.db.close() + assert admin_users._delete_user_rows_sync(user_id=ids[0]) is True + with env.factory() as db: + assert db.get(User, ids[0]) is None + assert db.get(Model, ids[2]) is not None + assert db.query(AutoModelCandidate).filter_by(config_id=ids[3]).count() == int( + retained_grant + ) + assert db.get(AutoModelConfig, ids[3]).fallback_model_id == ( + ids[2] if retained_grant else None + ) + assert db.get(Model, ids[4]).abilities == ( + ["chat", "tool_calling"] if retained_grant else ["chat"] + ) + + +def test_compatible_ability_edit_refreshes_auto_without_locking_credentials(env): + t = target(env, env.consumer, abilities=["chat", "tool_calling"]) + cfg = configure(env, env.consumer, [("text-a", t)]) + asyncio.run( + model_api.update_model( + t.model_id, + ModelUpdate( + abilities=["chat"], api_key="new-fake-key", base_url=t.base_url + ), + db=env.db, + user=env.consumer, + ) + ) + env.db.expire_all() + assert env.db.get(Model, cfg.router_model_id).abilities == ["chat"] + assert t.api_key == "new-fake-key" + + +def test_pruning_candidate_recalculates_remaining_abilities(env): + shared = target(env, env.owner, name="shared", shared=True) + own = target(env, env.consumer, name="own", abilities=["chat", "tool_calling"]) + cfg = configure(env, env.consumer, [("text-a", shared), ("text-b", own)]) + assert "tool_calling" not in cfg.router_model.abilities + asyncio.run( + model_api.update_model( + shared.model_id, + ModelUpdate(share_with_users=False), + db=env.db, + user=env.owner, + ) + ) + env.db.expire_all() + assert env.db.get(Model, cfg.router_model_id).abilities == ["chat", "tool_calling"] + + +def test_editing_legacy_unknown_profile_returns_conflict(env): + t = target(env, env.consumer) + cfg = configure(env, env.consumer, [("text-a", t)]) + cfg.candidates[0].routing_model_id = "removed-profile" + env.db.commit() + with pytest.raises(HTTPException) as error: + asyncio.run( + model_api.update_model( + t.model_id, + ModelUpdate(abilities=["chat", "tool_calling"]), + db=env.db, + user=env.consumer, + ) + ) + assert error.value.status_code == 409 + with pytest.raises(AutoModelUnavailableError, match="Unknown Auto profile"): + UserAwareModelStorage(env.db).get_llm_by_id( + cfg.router_model.model_id, env.consumer.id + ) diff --git a/tests/web/services/test_builder_chat_runtime.py b/tests/web/services/test_builder_chat_runtime.py index d63df290b5..ddbb7380c5 100644 --- a/tests/web/services/test_builder_chat_runtime.py +++ b/tests/web/services/test_builder_chat_runtime.py @@ -55,9 +55,15 @@ def get_llm_by_name_with_access( return selected_llm def get_configured_defaults( - self, user_id: int | None = None + self, + user_id: int | None = None, + *, + config_types: tuple[str, ...], + fallback_llm: object, ) -> tuple[None, None, None, object]: assert user_id == 42 + assert config_types == ("compact",) + assert fallback_llm is selected_llm return None, None, None, default_compact_llm monkeypatch.setattr( diff --git a/tests/web/services/test_task_orchestrator.py b/tests/web/services/test_task_orchestrator.py index cd62784ae0..d4aed09c1a 100644 --- a/tests/web/services/test_task_orchestrator.py +++ b/tests/web/services/test_task_orchestrator.py @@ -71,12 +71,16 @@ inspect_user_message_delivery, mark_user_message_delivery, ) -from xagent.web.services.client_error_messages import CLIENT_SAFE_TASK_FAILURE +from xagent.web.services.client_error_messages import ( + CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE, + CLIENT_SAFE_TASK_FAILURE, +) from xagent.web.services.connector_runtime import ( get_ephemeral_runtime_values, pop_ephemeral_runtime_values, store_ephemeral_runtime_values, ) +from xagent.web.services.llm_utils import AutoModelUnavailableError from xagent.web.services.mcp_runtime import ( MCPBuiltinOAuthActorPolicy, MCPBuiltinOAuthActorPolicyRequiredError, @@ -4199,3 +4203,32 @@ async def test_incidental_failure_persists_the_generic_history_type( assert settled["client_message_type"] == TASK_FAILURE_MESSAGE_TYPE assert settled["client_error_message"] == CLIENT_SAFE_TASK_FAILURE assert "secret-token-xyz" not in settled["client_error_message"] + + +@pytest.mark.asyncio +async def test_leased_auto_failure_preserves_client_classification(db_session) -> None: + error = AutoModelUnavailableError("private model binding details") + with _captured_terminal_broadcast(error, db_session) as ( + task_id, + frames, + settlements, + ): + task = db_session.query(Task).filter(Task.id == task_id).one() + with patch( + "xagent.web.api.websocket.execute_task_background", + new=AsyncMock(side_effect=error), + ) as execute: + await _run_failing_turn(task_id, int(task.user_id), task.source) + execute.assert_awaited_once() + assert execute.await_args.kwargs["task_lease"] == TaskLease( + task_id=task_id, runner_id="runner-a", run_id="run-a" + ) + + assert len(frames) == 1 + assert frames[0]["code"] == "auto_model_unavailable" + assert frames[0]["message"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + assert len(settlements) == 1 + assert settlements[0]["client_message_type"] == CLIENT_SAFE_FAILURE_MESSAGE_TYPE + assert settlements[0]["client_error_message"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + assert settlements[0]["error_message"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + assert "private model binding details" not in json.dumps(frames) diff --git a/tests/web/test_feishu_message_queue.py b/tests/web/test_feishu_message_queue.py index 333f5ca3c8..62b3b8b214 100644 --- a/tests/web/test_feishu_message_queue.py +++ b/tests/web/test_feishu_message_queue.py @@ -5,6 +5,8 @@ from xagent.web.channels.feishu.bot import FeishuBotInstance, FeishuChannelManager from xagent.web.models.task import TaskStatus +from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE +from xagent.web.services.llm_utils import AutoModelUnavailableError from xagent.web.services.task_execution_context_service import ( TaskExecutionRecoverySnapshot, ) @@ -30,16 +32,25 @@ def make_bot() -> FeishuBotInstance: @pytest.mark.asyncio +@pytest.mark.parametrize("auto_unavailable", [False, True]) async def test_error_after_prepare_settles_preclaimed_task_instead_of_orphaning_it( monkeypatch: pytest.MonkeyPatch, + auto_unavailable: bool, ) -> None: bot = object.__new__(FeishuBotInstance) bot.channel_id = 1 bot.channel_name = "Feishu prepare failure" bot.active_tasks = {} bot.api_client = object() - bot._save_active_tasks = lambda: (_ for _ in ()).throw( - RuntimeError("mapping persistence failed") + bot._save_active_tasks = lambda: None + failure = ( + AutoModelUnavailableError("private model details") + if auto_unavailable + else RuntimeError("snapshot failed") + ) + monkeypatch.setattr( + "xagent.web.channels.feishu.bot.load_task_setup_snapshot_sync", + lambda *_args: (_ for _ in ()).throw(failure), ) lease = TaskLease(task_id=45, runner_id="runner-a", run_id="run-a") finalized: list[TaskStatus] = [] @@ -95,7 +106,11 @@ async def send_text(_chat_id: str, text: str) -> str: assert finalized == [TaskStatus.FAILED] assert managed.closed is True - assert sent_messages == ["Sorry, an error occurred while processing your request."] + assert sent_messages == [ + CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE + if auto_unavailable + else "Sorry, an error occurred while processing your request." + ] @pytest.mark.asyncio diff --git a/tests/web/test_model_api.py b/tests/web/test_model_api.py index 28b1bece24..8349cd06a6 100644 --- a/tests/web/test_model_api.py +++ b/tests/web/test_model_api.py @@ -3,6 +3,7 @@ import asyncio import os import tempfile +from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch from urllib.parse import quote @@ -496,7 +497,7 @@ class TestModelAPI: """Test model management API endpoints""" def test_auto_config_binds_existing_models_and_blocks_candidate_delete( - self, test_db, regular_user, regular_headers, sample_model_data + self, test_db, regular_user, regular_headers, sample_model_data, monkeypatch ): first = client.post( "/api/models/", @@ -522,6 +523,18 @@ class Catalog: def known_model_ids(): return ("openai/gpt-5.5", "deepseek/deepseek-v4-flash") + @staticmethod + def get(profile_id): + return SimpleNamespace( + input_modalities=("text", "image") + if profile_id == "openai/gpt-5.5" + else ("text",) + ) + + monkeypatch.setattr( + "xagent.web.services.auto_model_service.load_router_profile_catalog", + lambda: Catalog(), + ) with patch( "xagent.web.services.auto_model_service.load_router_profile_catalog", return_value=Catalog(), diff --git a/tests/web/test_slack_channel.py b/tests/web/test_slack_channel.py index 471b204093..741ab7c890 100644 --- a/tests/web/test_slack_channel.py +++ b/tests/web/test_slack_channel.py @@ -41,6 +41,8 @@ from xagent.web.models.user_channel import SlackOAuthFlowState, UserChannel from xagent.web.schemas.user_channel import UserChannelCreate, UserChannelUpdate from xagent.web.services.channel_runtime import ChannelConfigSnapshot +from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE +from xagent.web.services.llm_utils import AutoModelUnavailableError from xagent.web.services.task_execution_context_service import ( TaskExecutionRecoverySnapshot, ) @@ -1205,8 +1207,10 @@ def test_slack_only_handles_mentions_in_shared_channels() -> None: @pytest.mark.asyncio -async def test_successful_slack_turn_reuses_channel_runtime( +@pytest.mark.parametrize("auto_unavailable", [False, True]) +async def test_slack_turn_reuses_channel_runtime_and_reports_auto_failure( monkeypatch: pytest.MonkeyPatch, + auto_unavailable: bool, ) -> None: bot = make_bot() bot._save_active_tasks = lambda: None # type: ignore[method-assign] @@ -1266,6 +1270,8 @@ def add_handler(self, handler: Any) -> None: class FakeAgentManager: async def get_agent_for_task(self, *_args: Any, **_kwargs: Any) -> Any: + if auto_unavailable: + raise AutoModelUnavailableError("private model details") return agent_service async def execute_task(self, **_kwargs: Any) -> dict[str, Any]: @@ -1284,6 +1290,8 @@ async def send_text( thread_ts: str | None, ) -> str: assert thread_ts is None + if auto_unavailable: + final_messages.append({"text": _text}) return "loading-ts" async def send_final_text(**kwargs: Any) -> None: @@ -1327,6 +1335,11 @@ async def send_final_text(**kwargs: Any) -> None: ) assert bot.active_tasks == {"T1:D1:U1:direct": 45} + if auto_unavailable: + assert finalized == [(TaskStatus.FAILED, "")] + assert final_messages == [{"text": CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE}] + assert managed.closed is True + return assert persisted[0]["content"] == "hello" assert finalized == [(TaskStatus.COMPLETED, "Slack reply")] assert finalized_execution_results == [execution_result]