workflow: inference hardcoded endpoint improvements

demo/workflow_vlm_chat/run.py

@@ -0,0 +1,13 @@
+"""Ask a vision-language model about an image.
+
+Drop in an image, type a question, run. The VLM answers in text. Uses the
+`chat_completion` schema so the workflow works with any image-text-to-text
+model regardless of which Inference Provider serves it.
+"""
+
+import gradio as gr
+
+demo = gr.Workflow(graph="workflow.json")
+
+if __name__ == "__main__":
+    demo.launch()

demo/workflow_vlm_chat/workflow.json

@@ -0,0 +1 @@
+{"schema_version":"2","name":"Ask About an Image","runtime":{"default":"client"},"references":[{"label":"Text","inputs":[{"id":"in","label":"Text","type":"text"}],"outputs":[{"id":"out","label":"Text","type":"text"}],"width":220,"height":163,"asset_type":"text","role":"reference","id":"81cf0eab-18c0-4d14-a2ce-217e2939ff2f","x":78.4375,"y":358.40234375,"data":{"in":"","out":""}}],"operators":[{"id":"op-vlm","label":"Kimi-K3","role":"operator","kind":"model","model_id":"moonshotai/Kimi-K3","endpoint":"chat_completion","pipeline_tag":"image-text-to-text","inputs":[{"id":"image","label":"Image","type":"image","required":false},{"id":"text","label":"Prompt","type":"text","required":false}],"outputs":[{"id":"out_0","label":"Text","type":"text","output_index":0}],"width":260,"height":184,"x":380,"y":180,"data":{"out_0":"Looks like you're tucked in somewhere cozy — maybe a van or camper? I can see wood paneling, some overhead lights, and what looks like bedding. Gives off late-night, settled-in-for-the-evening vibes.\n\nWhat's up with you? Just chilling, or is something on your mind?","text":"what going on "}}],"subjects":[{"id":"subj-answer","label":"Answer","role":"subject","asset_type":"text","inputs":[{"id":"in","label":"Text","type":"text"}],"outputs":[{"id":"out","label":"Text","type":"text"}],"width":320,"height":182,"x":720,"y":180,"data":{"in":"Looks like you're tucked in somewhere cozy — maybe a van or camper? I can see wood paneling, some overhead lights, and what looks like bedding. Gives off late-night, settled-in-for-the-evening vibes.\n\nWhat's up with you? Just chilling, or is something on your mind?"}}],"edges":[{"id":"e3","from_node_id":"op-vlm","from_port_id":"out_0","to_node_id":"subj-answer","to_port_id":"in","type":"text"},{"from_node_id":"81cf0eab-18c0-4d14-a2ce-217e2939ff2f","from_port_id":"out","to_node_id":"op-vlm","to_port_id":"text","type":"text","id":"8e33634e-9bba-4457-8138-76dff0c5ef35"}],"view":{"default":"canvas"}}

gradio/workflow.py

@@ -34,6 +34,10 @@
 from gradio.oauth import OAuthProfile, OAuthToken
 from gradio.route_utils import Request
 from gradio.utils import colab_check, get_space
+from gradio.workflow_provider_shims import (
+    PROVIDER_TASK_MISMATCH_RE,
+    run_via_helper,
+)
 
 if TYPE_CHECKING:
     from gradio.workflow_api import WorkflowEndpointManager
@@ -885,17 +889,15 @@ def process_item(item):
     "text-to-video": "text_to_video",
     "image-to-image": "image_to_image",
     "image-to-video": "image_to_video",
+    "image-text-to-video": "image_to_video",
     "image-classification": "image_classification",
     "object-detection": "object_detection",
     "image-segmentation": "image_segmentation",
-    "image-to-text": "image_to_text",
     "automatic-speech-recognition": "automatic_speech_recognition",
     "audio-classification": "audio_classification",
-    "visual-question-answering": "visual_question_answering",
-    "document-question-answering": "document_question_answering",
-    # Not visual_question_answering: the Hub routes every image-text-to-text
-    # model as `conversational`, and no provider serves the VQA task at all,
-    # so a task-specific call fails for every model carrying this tag.
+    "image-to-text": "chat_completion",
+    "visual-question-answering": "chat_completion",
+    "document-question-answering": "chat_completion",
     "image-text-to-text": "chat_completion",
 }
 
@@ -1006,6 +1008,20 @@ def _dispatch_model_endpoint(client, endpoint: str, kwargs: dict) -> str:
                 f"{model_name} returned no text (finish_reason={finish_reason})."
             )
         return json.dumps([text])
+    try:
+        fn_params = set(inspect.signature(fn).parameters)
+    except (TypeError, ValueError):
+        fn_params = set()
+    if "extra_body" in fn_params:
+        known: dict = {}
+        extras: dict = {}
+        for k, v in clean.items():
+            (known if k in fn_params else extras)[k] = v
+        if extras:
+            clean = {
+                **known,
+                "extra_body": {**(known.get("extra_body") or {}), **extras},
+            }
     if endpoint == "text_generation":
         clean.setdefault("max_new_tokens", 512)
         try:
@@ -1021,7 +1037,30 @@ def _dispatch_model_endpoint(client, endpoint: str, kwargs: dict) -> str:
             else:
                 raise
     else:
-        result = fn(**clean)
+        try:
+            result = fn(**clean)
+        except ValueError as exc:
+            m = PROVIDER_TASK_MISMATCH_RE.search(str(exc))
+            if not m:
+                raise
+            from huggingface_hub.inference._providers import get_provider_helper
+
+            helper = get_provider_helper(
+                m.group(2),  # ty: ignore[invalid-argument-type]
+                task=m.group(1),
+                model=client.model,
+            )
+            helper.task = m.group(3)
+            result = run_via_helper(client, helper, fn, endpoint, clean)
+        except KeyError:
+            if endpoint != "image_to_video":
+                raise
+            from huggingface_hub.inference._providers import get_provider_helper
+
+            helper = get_provider_helper(
+                client.provider, task="image-to-video", model=client.model
+            )
+            result = run_via_helper(client, helper, fn, endpoint, clean)
     ext = _ENDPOINT_OUTPUT_EXT.get(endpoint)
     if ext:
         return json.dumps([_save_tmp(result, ext)])
@@ -1080,7 +1119,9 @@ def call_model(
         client = InferenceClient(model=model_id, token=hf_token, provider=provider)
         args = json.loads(args_json)
         if isinstance(args, dict):
-            endpoint = pipeline_tag or ""
+            endpoint = (
+                _PIPELINE_TAG_TO_ENDPOINT.get(pipeline_tag or "") or pipeline_tag or ""
+            )
             return _dispatch_model_endpoint(client, endpoint, args)
 
         task = pipeline_tag or "text-generation"
@@ -1114,20 +1155,20 @@ def call_model(
             }
             return _dispatch_model_endpoint(client, endpoint, kwargs)
 
-        # Fallback for tasks not handled above: chat_completion (works for most
-        # text models across providers), then a raw POST as last resort.
-        try:
-            r = client.chat_completion(
-                [{"role": "user", "content": a0}], max_tokens=512
+        def _resolve(v):
+            return (
+                _img_url(v)
+                if isinstance(v, dict) and ("url" in v or "path" in v)
+                else v
             )
-            return json.dumps([r.choices[0].message.content])
-        except Exception:
-            pass
+
+        a1_missing = a1 is None or a1 == ""
+        payload = _resolve(a0) if a1_missing else [_resolve(a0), _resolve(a1)]
         headers = {"Authorization": f"Bearer {hf_token}"} if hf_token else {}
         fallback_resp = httpx.post(
             f"https://api-inference.huggingface.co/models/{model_id}",
             headers=headers,
-            json={"inputs": a0 if not a1 else [a0, a1]},
+            json={"inputs": payload},
             timeout=60,
         )
         fallback_resp.raise_for_status()

gradio/workflow_provider_shims.py

@@ -0,0 +1,53 @@
+"""Shims for `huggingface_hub` inference-provider client fragility.
+
+The per-task client methods hardcode both the task string and the response key
+path, so any drift in a provider's registration or response shape breaks every
+caller until upstream cuts a release. Prefer fixing upstream over adding here.
+"""
+
+from __future__ import annotations
+
+import inspect
+import re
+
+import httpx
+
+# Raised by `TaskProviderHelper._prepare_mapping_info` when the client method's
+# hardcoded task doesn't match the model's provider registration.
+PROVIDER_TASK_MISMATCH_RE = re.compile(
+    r"is not supported for task (\S+) and provider (\S+)\. "
+    r"Supported task: ([^.\s]+)\."
+)
+
+
+def fal_ai_video_fallback(helper, response, request_params) -> bytes:
+    from huggingface_hub.inference._providers.fal_ai import FalAIQueueTask
+
+    output = FalAIQueueTask.get_response(helper, response, request_params)
+    d = output if isinstance(output, dict) else {}
+    videos = d.get("videos") if isinstance(d.get("videos"), list) else None
+    url = (
+        (d.get("video") or {}).get("url")
+        or d.get("video_url")
+        or (videos[0].get("url") if videos else None)
+    )
+    if not url:
+        raise ValueError(f"Unexpected fal-ai response shape: {output}")
+    return httpx.get(url, timeout=60).content
+
+
+def run_via_helper(client, helper, fn, endpoint: str, clean: dict):
+    from huggingface_hub.inference._providers.fal_ai import FalAIQueueTask
+
+    input_key = next(iter(inspect.signature(fn).parameters), "inputs")
+    req = helper.prepare_request(
+        inputs=clean.pop(input_key, None),
+        parameters=clean,
+        headers=client.headers,
+        model=client.model,
+        api_key=client.token,
+    )
+    response = client._inner_post(req)
+    if endpoint == "image_to_video" and isinstance(helper, FalAIQueueTask):
+        return fal_ai_video_fallback(helper, response, req)
+    return helper.get_response(response, req)

js/workflowcanvas/workflow/WorkflowCanvas.svelte

@@ -144,12 +144,13 @@
 			!oauthHintShown &&
 			auth.isHFSpace &&
 			auth.writeAccessKnown &&
+			auth.oauthAvailableKnown &&
 			!auth.canWrite &&
 			!auth.oauthAvailable
 		) {
 			oauthHintShown = true;
 			showToast(
-				"Sign-in has not beed enabled on this Space. The author should add `hf_oauth: true` to the README so users can run workflows on their own inference quota, and authors can edit.",
+				"Sign-in has not been enabled on this Space. The author should add `hf_oauth: true` to the README so users can run workflows on their own inference quota, and authors can edit.",
 				0,
 				"warning"
 			);