diff --git a/demo/workflow_vlm_chat/run.py b/demo/workflow_vlm_chat/run.py new file mode 100644 index 00000000000..dbfd3023a7e --- /dev/null +++ b/demo/workflow_vlm_chat/run.py @@ -0,0 +1,13 @@ +"""Ask a vision-language model about an image. + +Drop in an image, type a question, run. The VLM answers in text. Uses the +`chat_completion` schema so the workflow works with any image-text-to-text +model regardless of which Inference Provider serves it. +""" + +import gradio as gr + +demo = gr.Workflow(graph="workflow.json") + +if __name__ == "__main__": + demo.launch() diff --git a/demo/workflow_vlm_chat/workflow.json b/demo/workflow_vlm_chat/workflow.json new file mode 100644 index 00000000000..0ece66602ad --- /dev/null +++ b/demo/workflow_vlm_chat/workflow.json @@ -0,0 +1 @@ +{"schema_version":"2","name":"Ask About an Image","runtime":{"default":"client"},"references":[{"label":"Text","inputs":[{"id":"in","label":"Text","type":"text"}],"outputs":[{"id":"out","label":"Text","type":"text"}],"width":220,"height":163,"asset_type":"text","role":"reference","id":"81cf0eab-18c0-4d14-a2ce-217e2939ff2f","x":78.4375,"y":358.40234375,"data":{"in":"","out":""}}],"operators":[{"id":"op-vlm","label":"Kimi-K3","role":"operator","kind":"model","model_id":"moonshotai/Kimi-K3","endpoint":"chat_completion","pipeline_tag":"image-text-to-text","inputs":[{"id":"image","label":"Image","type":"image","required":false},{"id":"text","label":"Prompt","type":"text","required":false}],"outputs":[{"id":"out_0","label":"Text","type":"text","output_index":0}],"width":260,"height":184,"x":380,"y":180,"data":{"out_0":"Looks like you're tucked in somewhere cozy — maybe a van or camper? I can see wood paneling, some overhead lights, and what looks like bedding. Gives off late-night, settled-in-for-the-evening vibes.\n\nWhat's up with you? Just chilling, or is something on your mind?","text":"what going on "}}],"subjects":[{"id":"subj-answer","label":"Answer","role":"subject","asset_type":"text","inputs":[{"id":"in","label":"Text","type":"text"}],"outputs":[{"id":"out","label":"Text","type":"text"}],"width":320,"height":182,"x":720,"y":180,"data":{"in":"Looks like you're tucked in somewhere cozy — maybe a van or camper? I can see wood paneling, some overhead lights, and what looks like bedding. Gives off late-night, settled-in-for-the-evening vibes.\n\nWhat's up with you? Just chilling, or is something on your mind?"}}],"edges":[{"id":"e3","from_node_id":"op-vlm","from_port_id":"out_0","to_node_id":"subj-answer","to_port_id":"in","type":"text"},{"from_node_id":"81cf0eab-18c0-4d14-a2ce-217e2939ff2f","from_port_id":"out","to_node_id":"op-vlm","to_port_id":"text","type":"text","id":"8e33634e-9bba-4457-8138-76dff0c5ef35"}],"view":{"default":"canvas"}} \ No newline at end of file diff --git a/gradio/workflow.py b/gradio/workflow.py index 65b8b48bfd1..e1b29d34353 100644 --- a/gradio/workflow.py +++ b/gradio/workflow.py @@ -34,6 +34,10 @@ from gradio.oauth import OAuthProfile, OAuthToken from gradio.route_utils import Request from gradio.utils import colab_check, get_space +from gradio.workflow_provider_shims import ( + PROVIDER_TASK_MISMATCH_RE, + run_via_helper, +) if TYPE_CHECKING: from gradio.workflow_api import WorkflowEndpointManager @@ -885,17 +889,15 @@ def process_item(item): "text-to-video": "text_to_video", "image-to-image": "image_to_image", "image-to-video": "image_to_video", + "image-text-to-video": "image_to_video", "image-classification": "image_classification", "object-detection": "object_detection", "image-segmentation": "image_segmentation", - "image-to-text": "image_to_text", "automatic-speech-recognition": "automatic_speech_recognition", "audio-classification": "audio_classification", - "visual-question-answering": "visual_question_answering", - "document-question-answering": "document_question_answering", - # Not visual_question_answering: the Hub routes every image-text-to-text - # model as `conversational`, and no provider serves the VQA task at all, - # so a task-specific call fails for every model carrying this tag. + "image-to-text": "chat_completion", + "visual-question-answering": "chat_completion", + "document-question-answering": "chat_completion", "image-text-to-text": "chat_completion", } @@ -1006,6 +1008,20 @@ def _dispatch_model_endpoint(client, endpoint: str, kwargs: dict) -> str: f"{model_name} returned no text (finish_reason={finish_reason})." ) return json.dumps([text]) + try: + fn_params = set(inspect.signature(fn).parameters) + except (TypeError, ValueError): + fn_params = set() + if "extra_body" in fn_params: + known: dict = {} + extras: dict = {} + for k, v in clean.items(): + (known if k in fn_params else extras)[k] = v + if extras: + clean = { + **known, + "extra_body": {**(known.get("extra_body") or {}), **extras}, + } if endpoint == "text_generation": clean.setdefault("max_new_tokens", 512) try: @@ -1021,7 +1037,30 @@ def _dispatch_model_endpoint(client, endpoint: str, kwargs: dict) -> str: else: raise else: - result = fn(**clean) + try: + result = fn(**clean) + except ValueError as exc: + m = PROVIDER_TASK_MISMATCH_RE.search(str(exc)) + if not m: + raise + from huggingface_hub.inference._providers import get_provider_helper + + helper = get_provider_helper( + m.group(2), # ty: ignore[invalid-argument-type] + task=m.group(1), + model=client.model, + ) + helper.task = m.group(3) + result = run_via_helper(client, helper, fn, endpoint, clean) + except KeyError: + if endpoint != "image_to_video": + raise + from huggingface_hub.inference._providers import get_provider_helper + + helper = get_provider_helper( + client.provider, task="image-to-video", model=client.model + ) + result = run_via_helper(client, helper, fn, endpoint, clean) ext = _ENDPOINT_OUTPUT_EXT.get(endpoint) if ext: return json.dumps([_save_tmp(result, ext)]) @@ -1080,7 +1119,9 @@ def call_model( client = InferenceClient(model=model_id, token=hf_token, provider=provider) args = json.loads(args_json) if isinstance(args, dict): - endpoint = pipeline_tag or "" + endpoint = ( + _PIPELINE_TAG_TO_ENDPOINT.get(pipeline_tag or "") or pipeline_tag or "" + ) return _dispatch_model_endpoint(client, endpoint, args) task = pipeline_tag or "text-generation" @@ -1114,20 +1155,20 @@ def call_model( } return _dispatch_model_endpoint(client, endpoint, kwargs) - # Fallback for tasks not handled above: chat_completion (works for most - # text models across providers), then a raw POST as last resort. - try: - r = client.chat_completion( - [{"role": "user", "content": a0}], max_tokens=512 + def _resolve(v): + return ( + _img_url(v) + if isinstance(v, dict) and ("url" in v or "path" in v) + else v ) - return json.dumps([r.choices[0].message.content]) - except Exception: - pass + + a1_missing = a1 is None or a1 == "" + payload = _resolve(a0) if a1_missing else [_resolve(a0), _resolve(a1)] headers = {"Authorization": f"Bearer {hf_token}"} if hf_token else {} fallback_resp = httpx.post( f"https://api-inference.huggingface.co/models/{model_id}", headers=headers, - json={"inputs": a0 if not a1 else [a0, a1]}, + json={"inputs": payload}, timeout=60, ) fallback_resp.raise_for_status() diff --git a/gradio/workflow_provider_shims.py b/gradio/workflow_provider_shims.py new file mode 100644 index 00000000000..8256a0694cb --- /dev/null +++ b/gradio/workflow_provider_shims.py @@ -0,0 +1,53 @@ +"""Shims for `huggingface_hub` inference-provider client fragility. + +The per-task client methods hardcode both the task string and the response key +path, so any drift in a provider's registration or response shape breaks every +caller until upstream cuts a release. Prefer fixing upstream over adding here. +""" + +from __future__ import annotations + +import inspect +import re + +import httpx + +# Raised by `TaskProviderHelper._prepare_mapping_info` when the client method's +# hardcoded task doesn't match the model's provider registration. +PROVIDER_TASK_MISMATCH_RE = re.compile( + r"is not supported for task (\S+) and provider (\S+)\. " + r"Supported task: ([^.\s]+)\." +) + + +def fal_ai_video_fallback(helper, response, request_params) -> bytes: + from huggingface_hub.inference._providers.fal_ai import FalAIQueueTask + + output = FalAIQueueTask.get_response(helper, response, request_params) + d = output if isinstance(output, dict) else {} + videos = d.get("videos") if isinstance(d.get("videos"), list) else None + url = ( + (d.get("video") or {}).get("url") + or d.get("video_url") + or (videos[0].get("url") if videos else None) + ) + if not url: + raise ValueError(f"Unexpected fal-ai response shape: {output}") + return httpx.get(url, timeout=60).content + + +def run_via_helper(client, helper, fn, endpoint: str, clean: dict): + from huggingface_hub.inference._providers.fal_ai import FalAIQueueTask + + input_key = next(iter(inspect.signature(fn).parameters), "inputs") + req = helper.prepare_request( + inputs=clean.pop(input_key, None), + parameters=clean, + headers=client.headers, + model=client.model, + api_key=client.token, + ) + response = client._inner_post(req) + if endpoint == "image_to_video" and isinstance(helper, FalAIQueueTask): + return fal_ai_video_fallback(helper, response, req) + return helper.get_response(response, req) diff --git a/js/workflowcanvas/workflow/WorkflowCanvas.svelte b/js/workflowcanvas/workflow/WorkflowCanvas.svelte index d4d05220edf..a2132100a34 100644 --- a/js/workflowcanvas/workflow/WorkflowCanvas.svelte +++ b/js/workflowcanvas/workflow/WorkflowCanvas.svelte @@ -144,12 +144,13 @@ !oauthHintShown && auth.isHFSpace && auth.writeAccessKnown && + auth.oauthAvailableKnown && !auth.canWrite && !auth.oauthAvailable ) { oauthHintShown = true; showToast( - "Sign-in has not beed enabled on this Space. The author should add `hf_oauth: true` to the README so users can run workflows on their own inference quota, and authors can edit.", + "Sign-in has not been enabled on this Space. The author should add `hf_oauth: true` to the README so users can run workflows on their own inference quota, and authors can edit.", 0, "warning" ); diff --git a/js/workflowcanvas/workflow/WorkflowNodeSF.svelte b/js/workflowcanvas/workflow/WorkflowNodeSF.svelte index 40313a80201..84cc0b91ce1 100644 --- a/js/workflowcanvas/workflow/WorkflowNodeSF.svelte +++ b/js/workflowcanvas/workflow/WorkflowNodeSF.svelte @@ -1,6 +1,12 @@