From cbaa16e7b98503298e8972206f851993a85d85e4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 20 Aug 2026 16:29:31 +0000 Subject: [PATCH 1/2] fix(cli): show eval input upload progress Co-authored-by: Blaine Kasten --- src/together/lib/cli/api/evals/create.py | 11 ++- tests/cli/test_evals.py | 107 ++++++++++++++++++++++- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/src/together/lib/cli/api/evals/create.py b/src/together/lib/cli/api/evals/create.py index 345794160..be83e0c08 100644 --- a/src/together/lib/cli/api/evals/create.py +++ b/src/together/lib/cli/api/evals/create.py @@ -9,6 +9,7 @@ from together._utils._json import openapi_dumps from together.lib.cli.utils.config import CLIConfigParameter from together.lib.cli.utils._console import console +from together.lib.cli.components.upload_progress import upload_file_with_progress from together.types.eval_create_params import ( ParametersEvaluationScoreParameters, ParametersEvaluationCompareParameters, @@ -140,7 +141,15 @@ async def create( # If the user passes a path to a file, try to upload it to the files API first # Uploads are idempotent so we can depend on this API always giving us a file ID if _check_path_exists(input_data_file_path): - file_upload = await config.client.files.upload(Path(input_data_file_path), purpose="eval", check=False) + input_data_path = Path(input_data_file_path) + file_upload = await upload_file_with_progress( + config.client.files.upload, + input_data_path, + enabled=not config.json, + description=f"Uploading eval input file {input_data_path.name}", + purpose="eval", + check=False, + ) training_file = file_upload.id else: training_file = input_data_file_path diff --git a/tests/cli/test_evals.py b/tests/cli/test_evals.py index 0bc318f17..52c3f82a5 100644 --- a/tests/cli/test_evals.py +++ b/tests/cli/test_evals.py @@ -2,7 +2,9 @@ import os import json -from typing import cast +from typing import Any, cast +from pathlib import Path +from unittest.mock import AsyncMock, patch import httpx import pytest @@ -10,6 +12,7 @@ from respx.models import Call from tests.cli.utils import CliRunner +from together.types.file_response import FileResponse base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") API_KEY = "0000000000000000000000000000000000000000" @@ -27,6 +30,23 @@ _EVAL_STATUS = {"status": "completed", "results": None} +def _file_response(**kwargs: Any) -> FileResponse: + defaults: dict[str, Any] = { + "id": "file-up", + "bytes": 10, + "created_at": 1, + "filename": "eval-input.jsonl", + "FileType": "jsonl", + "object": "file", + "Processed": True, + "purpose": "eval", + } + defaults.update(kwargs) + if hasattr(FileResponse, "model_validate"): + return FileResponse.model_validate(defaults) + return FileResponse.parse_obj(defaults) # pyright: ignore[reportDeprecated] + + class TestEvalsList: @pytest.mark.respx(base_url=base_url) def test_list_passes_status_and_limit(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: @@ -97,3 +117,88 @@ def test_compare_passes_disable_position_bias_correction( payload = json.loads(req.content) assert payload["type"] == "compare" assert payload["parameters"]["disable_position_bias_correction"] is True + + @pytest.mark.respx(base_url=base_url) + def test_local_input_upload_uses_progress_callback( + self, tmp_path: Path, respx_mock: MockRouter, cli_runner: CliRunner + ) -> None: + input_file = tmp_path / "eval-input.jsonl" + input_file.write_text('{"prompt": "hello", "response_a": "a", "response_b": "b"}\n') + uploaded = _file_response(id="file-uploaded") + route = respx_mock.post("/evaluation").mock( + return_value=httpx.Response(200, json={"workflow_id": "eval-wf-1", "status": "pending"}) + ) + + with patch("together.resources.files.AsyncFilesResource.upload", new_callable=AsyncMock) as upload_mock: + upload_mock.return_value = uploaded + result = cli_runner.invoke( + [ + "evals", + "create", + "--type", + "compare", + "--judge-model", + "Qwen/Qwen3.5-9B", + "--judge-model-source", + "serverless", + "--judge-system-template", + "Choose the better response.", + "--input-data-file-path", + str(input_file), + "--model-a-field", + "response_a", + "--model-b-field", + "response_b", + ] + ) + + assert result.exit_code == 0 + upload_mock.assert_called_once() + upload_kwargs = upload_mock.call_args.kwargs + assert upload_kwargs["file"] == input_file + assert upload_kwargs["purpose"] == "eval" + assert upload_kwargs["check"] is False + assert upload_kwargs["progress_callback"] is not None + req = cast(Call, route.calls[0]).request + payload = json.loads(req.content) + assert payload["parameters"]["input_data_file_path"] == "file-uploaded" + + @pytest.mark.respx(base_url=base_url) + def test_local_input_upload_json_mode_disables_progress_callback( + self, tmp_path: Path, respx_mock: MockRouter, cli_runner: CliRunner + ) -> None: + input_file = tmp_path / "eval-input.jsonl" + input_file.write_text('{"prompt": "hello", "response_a": "a", "response_b": "b"}\n') + uploaded = _file_response(id="file-uploaded") + respx_mock.post("/evaluation").mock( + return_value=httpx.Response(200, json={"workflow_id": "eval-wf-1", "status": "pending"}) + ) + + with patch("together.resources.files.AsyncFilesResource.upload", new_callable=AsyncMock) as upload_mock: + upload_mock.return_value = uploaded + result = cli_runner.invoke( + [ + "evals", + "create", + "--type", + "compare", + "--judge-model", + "Qwen/Qwen3.5-9B", + "--judge-model-source", + "serverless", + "--judge-system-template", + "Choose the better response.", + "--input-data-file-path", + str(input_file), + "--model-a-field", + "response_a", + "--model-b-field", + "response_b", + "--json", + ] + ) + + assert result.exit_code == 0 + upload_mock.assert_called_once() + assert upload_mock.call_args.kwargs["progress_callback"] is None + assert json.loads(result.out_out.lstrip("\n"))["workflow_id"] == "eval-wf-1" From 81978675e293ffbc6bf5e001a99e89410bfeab57 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 20 Aug 2026 16:30:13 +0000 Subject: [PATCH 2/2] style(cli): sort evals upload imports Co-authored-by: Blaine Kasten --- src/together/lib/cli/api/evals/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/together/lib/cli/api/evals/create.py b/src/together/lib/cli/api/evals/create.py index be83e0c08..99c59a312 100644 --- a/src/together/lib/cli/api/evals/create.py +++ b/src/together/lib/cli/api/evals/create.py @@ -9,7 +9,6 @@ from together._utils._json import openapi_dumps from together.lib.cli.utils.config import CLIConfigParameter from together.lib.cli.utils._console import console -from together.lib.cli.components.upload_progress import upload_file_with_progress from together.types.eval_create_params import ( ParametersEvaluationScoreParameters, ParametersEvaluationCompareParameters, @@ -22,6 +21,7 @@ ParametersEvaluationCompareParametersModelAEvaluationModelRequest, ParametersEvaluationCompareParametersModelBEvaluationModelRequest, ) +from together.lib.cli.components.upload_progress import upload_file_with_progress async def create(