diff --git a/.changeset/sad-stars-play.md b/.changeset/sad-stars-play.md new file mode 100644 index 0000000000..fdcb4ffd6b --- /dev/null +++ b/.changeset/sad-stars-play.md @@ -0,0 +1,6 @@ +--- +"@gradio/workflowcanvas": minor +"gradio": minor +--- + +feat:workflow: refactor InferenceClient endpoints logic diff --git a/demo/workflow_vlm_chat/run.py b/demo/workflow_vlm_chat/run.py new file mode 100644 index 0000000000..17f2af035f --- /dev/null +++ b/demo/workflow_vlm_chat/run.py @@ -0,0 +1,13 @@ +"""Ask a vision-language model about an image. + +Drop in an image, type a question, run — the VLM answers in text. Uses the +`chat_completion` schema so the workflow works with any image-text-to-text +model regardless of which Inference Provider serves it. +""" + +import gradio as gr + +demo = gr.Workflow(graph="workflow.json") + +if __name__ == "__main__": + demo.launch() diff --git a/demo/workflow_vlm_chat/workflow.json b/demo/workflow_vlm_chat/workflow.json new file mode 100644 index 0000000000..0ece66602a --- /dev/null +++ b/demo/workflow_vlm_chat/workflow.json @@ -0,0 +1 @@ +{"schema_version":"2","name":"Ask About an Image","runtime":{"default":"client"},"references":[{"label":"Text","inputs":[{"id":"in","label":"Text","type":"text"}],"outputs":[{"id":"out","label":"Text","type":"text"}],"width":220,"height":163,"asset_type":"text","role":"reference","id":"81cf0eab-18c0-4d14-a2ce-217e2939ff2f","x":78.4375,"y":358.40234375,"data":{"in":"","out":""}}],"operators":[{"id":"op-vlm","label":"Kimi-K3","role":"operator","kind":"model","model_id":"moonshotai/Kimi-K3","endpoint":"chat_completion","pipeline_tag":"image-text-to-text","inputs":[{"id":"image","label":"Image","type":"image","required":false},{"id":"text","label":"Prompt","type":"text","required":false}],"outputs":[{"id":"out_0","label":"Text","type":"text","output_index":0}],"width":260,"height":184,"x":380,"y":180,"data":{"out_0":"Looks like you're tucked in somewhere cozy — maybe a van or camper? I can see wood paneling, some overhead lights, and what looks like bedding. Gives off late-night, settled-in-for-the-evening vibes.\n\nWhat's up with you? Just chilling, or is something on your mind?","text":"what going on "}}],"subjects":[{"id":"subj-answer","label":"Answer","role":"subject","asset_type":"text","inputs":[{"id":"in","label":"Text","type":"text"}],"outputs":[{"id":"out","label":"Text","type":"text"}],"width":320,"height":182,"x":720,"y":180,"data":{"in":"Looks like you're tucked in somewhere cozy — maybe a van or camper? I can see wood paneling, some overhead lights, and what looks like bedding. Gives off late-night, settled-in-for-the-evening vibes.\n\nWhat's up with you? Just chilling, or is something on your mind?"}}],"edges":[{"id":"e3","from_node_id":"op-vlm","from_port_id":"out_0","to_node_id":"subj-answer","to_port_id":"in","type":"text"},{"from_node_id":"81cf0eab-18c0-4d14-a2ce-217e2939ff2f","from_port_id":"out","to_node_id":"op-vlm","to_port_id":"text","type":"text","id":"8e33634e-9bba-4457-8138-76dff0c5ef35"}],"view":{"default":"canvas"}} \ No newline at end of file diff --git a/gradio/workflow.py b/gradio/workflow.py index 65b8b48bfd..102a422cee 100644 --- a/gradio/workflow.py +++ b/gradio/workflow.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import functools import inspect import json import logging @@ -655,193 +656,238 @@ def process_item(item): return _format_error(e) -_INFERENCE_ENDPOINT_SCHEMAS: dict[str, dict] = { - "text_to_image": { - "inputs": [ - {"id": "prompt", "label": "Prompt", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Image", "type": "image", "output_index": 0} - ], - }, - "text_to_speech": { - "inputs": [ - {"id": "text", "label": "Text", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Audio", "type": "audio", "output_index": 0} - ], - }, - "text_to_video": { - "inputs": [ - {"id": "prompt", "label": "Prompt", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Video", "type": "video", "output_index": 0} - ], - }, - "image_to_image": { - "inputs": [ - {"id": "image", "label": "Image", "type": "image"}, - {"id": "prompt", "label": "Prompt", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Image", "type": "image", "output_index": 0} - ], - }, - "image_to_video": { - "inputs": [ - {"id": "image", "label": "Image", "type": "image"}, - {"id": "prompt", "label": "Prompt", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Video", "type": "video", "output_index": 0} - ], - }, - "text_generation": { - "inputs": [ - {"id": "prompt", "label": "Prompt", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Text", "type": "text", "output_index": 0} - ], - }, - "summarization": { - "inputs": [ - {"id": "text", "label": "Text", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Summary", "type": "text", "output_index": 0} - ], - }, - "translation": { - "inputs": [ - {"id": "text", "label": "Text", "type": "text"}, - {"id": "src_lang", "label": "Source Language", "type": "text"}, - {"id": "tgt_lang", "label": "Target Language", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Translation", "type": "text", "output_index": 0} - ], - }, - "fill_mask": { - "inputs": [{"id": "text", "label": "Text", "type": "text"}], - "outputs": [ - {"id": "out_0", "label": "Result", "type": "json", "output_index": 0} - ], - }, - "text_classification": { - "inputs": [{"id": "text", "label": "Text", "type": "text"}], - "outputs": [ - {"id": "out_0", "label": "Labels", "type": "json", "output_index": 0} - ], - }, - "token_classification": { - "inputs": [{"id": "text", "label": "Text", "type": "text"}], - "outputs": [ - {"id": "out_0", "label": "Entities", "type": "json", "output_index": 0} - ], - }, - "zero_shot_classification": { - "inputs": [ - {"id": "text", "label": "Text", "type": "text"}, - {"id": "candidate_labels", "label": "Candidate Labels", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Scores", "type": "json", "output_index": 0} - ], - }, - "sentence_similarity": { - "inputs": [ - {"id": "sentence", "label": "Sentence", "type": "text"}, - {"id": "other_sentences", "label": "Other Sentences", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Scores", "type": "json", "output_index": 0} - ], - }, - "question_answering": { - "inputs": [ - {"id": "question", "label": "Question", "type": "text"}, - {"id": "context", "label": "Context", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Answer", "type": "text", "output_index": 0} - ], - }, - "feature_extraction": { - "inputs": [{"id": "text", "label": "Text", "type": "text"}], - "outputs": [ - {"id": "out_0", "label": "Embeddings", "type": "json", "output_index": 0} - ], - }, - "image_classification": { - "inputs": [{"id": "image", "label": "Image", "type": "image"}], - "outputs": [ - {"id": "out_0", "label": "Labels", "type": "json", "output_index": 0} - ], - }, - "object_detection": { - "inputs": [{"id": "image", "label": "Image", "type": "image"}], - "outputs": [ - {"id": "out_0", "label": "Detections", "type": "json", "output_index": 0} - ], - }, - "image_segmentation": { - "inputs": [{"id": "image", "label": "Image", "type": "image"}], - "outputs": [ - {"id": "out_0", "label": "Segments", "type": "json", "output_index": 0} - ], - }, - "image_to_text": { - "inputs": [{"id": "image", "label": "Image", "type": "image"}], - "outputs": [ - {"id": "out_0", "label": "Text", "type": "text", "output_index": 0} - ], - }, - "automatic_speech_recognition": { - "inputs": [{"id": "audio", "label": "Audio", "type": "audio"}], - "outputs": [ - {"id": "out_0", "label": "Text", "type": "text", "output_index": 0} - ], - }, - "audio_classification": { - "inputs": [{"id": "audio", "label": "Audio", "type": "audio"}], - "outputs": [ - {"id": "out_0", "label": "Labels", "type": "json", "output_index": 0} - ], - }, - "visual_question_answering": { - "inputs": [ - {"id": "image", "label": "Image", "type": "image"}, - {"id": "question", "label": "Question", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Answer", "type": "text", "output_index": 0} - ], - }, - "document_question_answering": { - "inputs": [ - {"id": "image", "label": "Document", "type": "image"}, - {"id": "question", "label": "Question", "type": "text"}, - ], - "outputs": [ - {"id": "out_0", "label": "Answer", "type": "text", "output_index": 0} - ], - }, - # Vision-language models are served as `conversational`, so they're called - # through chat completions rather than a task-specific endpoint. Port order - # matches the canvas's image-text-to-text template (image, then prompt). - "chat_completion": { +_PORT_TYPE_BY_PARAM: dict[str, str] = { + "image": "image", + "images": "image", + "document": "image", + "audio": "audio", + "video": "video", +} + +# Substring safety net — catches renames like `img` or `source_audio`. +_MEDIA_TOKEN_TO_TYPE: dict[str, str] = { + "image": "image", + "img": "image", + "images": "image", + "picture": "image", + "pic": "image", + "document": "image", + "audio": "audio", + "sound": "audio", + "video": "video", + "clip": "video", +} + +_OUTPUT_SUFFIX_TO_TYPE: dict[str, str] = { + "_to_image": "image", + "_to_speech": "audio", + "_to_audio": "audio", + "_to_video": "video", +} + +_MEDIA_EXT: dict[str, str] = {"image": "png", "audio": "wav", "video": "mp4"} + +_JSON_OUTPUT_METHODS: frozenset[str] = frozenset( + { + "text_classification", + "token_classification", + "zero_shot_classification", + "zero_shot_image_classification", + "image_classification", + "audio_classification", + "object_detection", + "image_segmentation", + "fill_mask", + "sentence_similarity", + "feature_extraction", + "tabular_classification", + "tabular_regression", + } +) + +# Semantic labels only — modality labels come from `_output_label`. +_OUTPUT_LABELS: dict[str, str] = { + "summarization": "Summary", + "translation": "Translation", + "question_answering": "Answer", + "visual_question_answering": "Answer", + "document_question_answering": "Answer", + "table_question_answering": "Answer", + "feature_extraction": "Embeddings", + "sentence_similarity": "Scores", + "text_classification": "Labels", + "zero_shot_classification": "Labels", + "zero_shot_image_classification": "Labels", + "image_classification": "Labels", + "audio_classification": "Labels", + "token_classification": "Entities", + "object_detection": "Detections", + "image_segmentation": "Segments", + "fill_mask": "Predictions", + "automatic_speech_recognition": "Transcript", +} + +_SKIP_PARAMS: frozenset[str] = frozenset( + { + "self", + "model", + "parameters", + "extra_headers", + "extra_body", + "stream", + "stream_options", + "return_type", + "return_dict", + "generate_parameters", + "generation_parameters", + "clean_up_tokenization_spaces", + "handle_impossible_answer", + "align_to_words", + "doc_stride", + "aggregation_strategy", + "ignore_labels", + "stride", + "function_to_apply", + "mask_threshold", + "overlap_mask_area_threshold", + "adapter_id", + "best_of", + "details", + "decoder_input_details", + "epsilon_cutoff", + "eta_cutoff", + "early_stopping", + "prompt_name", + "normalize", + "truncate", + "targets", + "sequential", + "hypothesis_template", + "decoder_start_token_id", + "forced_bos_token_id", + } +) + +_NON_TASK_METHODS: frozenset[str] = frozenset( + { + "chat_completion", + "close", + "post", + "health_check", + "list_deployed_models", + "get_endpoint_info", + "get_recommended_model", + "get_model_status", + "list_endpoints", + "conversational", + } +) + + +def _port_type(param_name: str, annotation: object) -> str: + if param_name in _PORT_TYPE_BY_PARAM: + return _PORT_TYPE_BY_PARAM[param_name] + tokens = set(param_name.lower().split("_")) + for token, port_type in _MEDIA_TOKEN_TO_TYPE.items(): + if token in tokens: + return port_type + origin = getattr(annotation, "__origin__", None) + if origin is Union or isinstance(annotation, types.UnionType): + args = [a for a in getattr(annotation, "__args__", ()) if a is not type(None)] + annotation = args[0] if len(args) == 1 else annotation + origin = getattr(annotation, "__origin__", annotation) + else: + origin = annotation + if origin in (int, float): + return "number" + if origin is bool: + return "boolean" + return "text" + + +def _output_port_type(method_name: str) -> str: + for suffix, port_type in _OUTPUT_SUFFIX_TO_TYPE.items(): + if method_name.endswith(suffix): + return port_type + if method_name in _JSON_OUTPUT_METHODS: + return "json" + return "text" + + +def _output_label(method_name: str, port_type: str) -> str: + if method_name in _OUTPUT_LABELS: + return _OUTPUT_LABELS[method_name] + if port_type in ("image", "audio", "video", "text"): + return port_type.capitalize() + return "Output" + + +@functools.lru_cache(maxsize=1) +def _inference_endpoint_schemas() -> dict[str, dict]: + from huggingface_hub import InferenceClient + + endpoints: dict[str, dict] = {} + for name in dir(InferenceClient): + if name.startswith("_") or name in _NON_TASK_METHODS: + continue + method = getattr(InferenceClient, name) + if not callable(method): + continue + try: + sig = inspect.signature(method) + except (TypeError, ValueError): + continue + inputs = [] + for pname, param in sig.parameters.items(): + if pname in _SKIP_PARAMS: + continue + if param.kind in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ): + continue + inputs.append( + { + "id": pname, + "label": pname.replace("_", " ").title(), + "type": _port_type(pname, param.annotation), + "required": param.default is inspect.Parameter.empty, + } + ) + if not inputs: + continue + port_type = _output_port_type(name) + endpoints[name] = { + "inputs": inputs, + "outputs": [ + { + "id": "out_0", + "label": _output_label(name, port_type), + "type": port_type, + "output_index": 0, + } + ], + } + # chat_completion needs a hand-crafted shape (image + prompt, not messages). + endpoints["chat_completion"] = { "inputs": [ - {"id": "image", "label": "Image", "type": "image"}, - {"id": "text", "label": "Prompt", "type": "text"}, + {"id": "image", "label": "Image", "type": "image", "required": False}, + {"id": "text", "label": "Prompt", "type": "text", "required": False}, ], "outputs": [ {"id": "out_0", "label": "Text", "type": "text", "output_index": 0} ], - }, -} + } + return endpoints + + +# Warm the schema cache at import time so it captures the real InferenceClient +# rather than whatever a test happens to have patched it with. Also moves the +# ~100ms introspection cost off the first request thread. +try: + _inference_endpoint_schemas() +except Exception: + pass # Generous by default: vision-language models are asked to emit whole files @@ -853,53 +899,27 @@ def process_item(item): _CHAT_MAX_TOKENS = 16384 -_ENDPOINT_OUTPUT_EXT: dict[str, str] = { - "text_to_image": "png", - "image_to_image": "png", - "text_to_speech": "wav", - "text_to_video": "mp4", - "image_to_video": "mp4", -} - - -# Legacy pipeline tags (as sent by older saved workflows and the browser -# executor) → InferenceClient endpoint names. depth-estimation is absent on -# purpose: InferenceClient has no such method, so it keeps its raw-POST branch -# in call_model. Unmapped tags fall through to the chat/raw-POST fallback. -_PIPELINE_TAG_TO_ENDPOINT: dict[str, str] = { - "text-generation": "text_generation", +# Only tags whose method name isn't `tag.replace("-", "_")`. Vision tags +# route through chat_completion — no Inference Provider serves the +# task-specific VQA/image-to-text endpoints. +_PIPELINE_TAG_ALIASES: dict[str, str] = { "text2text-generation": "text_generation", "conversational": "text_generation", - "summarization": "summarization", - "translation": "translation", - "fill-mask": "fill_mask", - "text-classification": "text_classification", - "token-classification": "token_classification", - "zero-shot-classification": "zero_shot_classification", - "sentence-similarity": "sentence_similarity", - "question-answering": "question_answering", - "feature-extraction": "feature_extraction", - "text-to-image": "text_to_image", - "text-to-speech": "text_to_speech", "text-to-audio": "text_to_speech", - "text-to-video": "text_to_video", - "image-to-image": "image_to_image", - "image-to-video": "image_to_video", - "image-classification": "image_classification", - "object-detection": "object_detection", - "image-segmentation": "image_segmentation", - "image-to-text": "image_to_text", - "automatic-speech-recognition": "automatic_speech_recognition", - "audio-classification": "audio_classification", - "visual-question-answering": "visual_question_answering", - "document-question-answering": "document_question_answering", - # Not visual_question_answering: the Hub routes every image-text-to-text - # model as `conversational`, and no provider serves the VQA task at all, - # so a task-specific call fails for every model carrying this tag. "image-text-to-text": "chat_completion", + "visual-question-answering": "chat_completion", + "document-question-answering": "chat_completion", + "image-to-text": "chat_completion", } +def _endpoint_for_tag(pipeline_tag: str | None) -> str | None: + if not pipeline_tag: + return None + name = _PIPELINE_TAG_ALIASES.get(pipeline_tag, pipeline_tag.replace("-", "_")) + return name if name in _inference_endpoint_schemas() else None + + # Client params that expect a list of strings; port values arrive as a single # string, split on the given pattern. _ENDPOINT_LIST_KWARGS: dict[str, dict[str, str]] = { @@ -911,16 +931,12 @@ def process_item(item): def get_model_endpoints( _data, _request: Optional[Request] = None, _token: Optional[OAuthToken] = None ) -> str: - from huggingface_hub import InferenceClient - - # Only advertise endpoints the installed huggingface_hub can actually run, - # so the UI never shapes a node around a method that would fail server-side. - endpoints = [ - {"name": name, **schema} - for name, schema in _INFERENCE_ENDPOINT_SCHEMAS.items() - if getattr(InferenceClient, name, None) is not None - ] - return json.dumps(endpoints) + return json.dumps( + [ + {"name": name, **schema} + for name, schema in _inference_endpoint_schemas().items() + ] + ) def _dispatch_model_endpoint(client, endpoint: str, kwargs: dict) -> str: @@ -935,7 +951,8 @@ def _dispatch_model_endpoint(client, endpoint: str, kwargs: dict) -> str: "`pip install -U huggingface_hub`." ) schema_ids = [ - p["id"] for p in _INFERENCE_ENDPOINT_SCHEMAS.get(endpoint, {}).get("inputs", []) + p["id"] + for p in _inference_endpoint_schemas().get(endpoint, {}).get("inputs", []) ] clean: dict = {} # Chat images are dereferenced by the provider, not locally, so they need a @@ -1006,6 +1023,26 @@ def _dispatch_model_endpoint(client, endpoint: str, kwargs: dict) -> str: f"{model_name} returned no text (finish_reason={finish_reason})." ) return json.dumps([text]) + # Kwargs outside the method's declared signature are custom-port extras + # (from the "+ Add param" UI). Fold them into `extra_body` — the escape + # hatch InferenceClient exposes for provider-specific params. Only split + # when the method actually accepts `extra_body`; otherwise pass all + # kwargs through (methods without extra_body will TypeError on genuinely + # unknown kwargs — the honest failure mode). + try: + fn_params = set(inspect.signature(fn).parameters) + except (TypeError, ValueError): + fn_params = set() + if "extra_body" in fn_params: + known: dict = {} + extras: dict = {} + for k, v in clean.items(): + (known if k in fn_params else extras)[k] = v + if extras: + clean = { + **known, + "extra_body": {**(known.get("extra_body") or {}), **extras}, + } if endpoint == "text_generation": clean.setdefault("max_new_tokens", 512) try: @@ -1022,7 +1059,7 @@ def _dispatch_model_endpoint(client, endpoint: str, kwargs: dict) -> str: raise else: result = fn(**clean) - ext = _ENDPOINT_OUTPUT_EXT.get(endpoint) + ext = _MEDIA_EXT.get(_output_port_type(endpoint)) if ext: return json.dumps([_save_tmp(result, ext)]) if isinstance(result, list) and result and hasattr(result[0], "answer"): @@ -1080,7 +1117,9 @@ def call_model( client = InferenceClient(model=model_id, token=hf_token, provider=provider) args = json.loads(args_json) if isinstance(args, dict): - endpoint = pipeline_tag or "" + # Dict args come from nodes with custom ports; fall through to + # treating pipeline_tag as an endpoint name if it doesn't alias. + endpoint = _endpoint_for_tag(pipeline_tag) or pipeline_tag or "" return _dispatch_model_endpoint(client, endpoint, args) task = pipeline_tag or "text-generation" @@ -1103,31 +1142,32 @@ def call_model( depth_img = _Image.open(_io.BytesIO(resp.content)) return json.dumps([_save_tmp(depth_img, "png")]) - endpoint = _PIPELINE_TAG_TO_ENDPOINT.get(task) + endpoint = _endpoint_for_tag(task) if endpoint: - # Positional args from legacy saved workflows and the browser - # executor map onto the endpoint schema's input order. - schema_inputs = _INFERENCE_ENDPOINT_SCHEMAS[endpoint]["inputs"] + schema_inputs = _inference_endpoint_schemas()[endpoint]["inputs"] kwargs = { schema_inputs[i]["id"]: val for i, val in enumerate(args[: len(schema_inputs)]) } return _dispatch_model_endpoint(client, endpoint, kwargs) - # Fallback for tasks not handled above: chat_completion (works for most - # text models across providers), then a raw POST as last resort. - try: - r = client.chat_completion( - [{"role": "user", "content": a0}], max_tokens=512 + # Unknown task — fall through to HF's raw inference API for server- + # side dispatch. `a1 == ""`/`None` means "not provided"; real falsy + # values (0, False) survive. + def _resolve(v): + return ( + _img_url(v) + if isinstance(v, dict) and ("url" in v or "path" in v) + else v ) - return json.dumps([r.choices[0].message.content]) - except Exception: - pass + + a1_missing = a1 is None or a1 == "" + payload = _resolve(a0) if a1_missing else [_resolve(a0), _resolve(a1)] headers = {"Authorization": f"Bearer {hf_token}"} if hf_token else {} fallback_resp = httpx.post( f"https://api-inference.huggingface.co/models/{model_id}", headers=headers, - json={"inputs": a0 if not a1 else [a0, a1]}, + json={"inputs": payload}, timeout=60, ) fallback_resp.raise_for_status() diff --git a/js/workflowcanvas/workflow/WorkflowNodeSF.svelte b/js/workflowcanvas/workflow/WorkflowNodeSF.svelte index 40313a8020..9d85b5c088 100644 --- a/js/workflowcanvas/workflow/WorkflowNodeSF.svelte +++ b/js/workflowcanvas/workflow/WorkflowNodeSF.svelte @@ -1,6 +1,12 @@