From c54cfcd73ede3502a8fba77d94f92eb8c41385ef Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Mon, 17 Aug 2026 08:36:13 +0000 Subject: [PATCH] Add music generation agent tool --- .env.example | 10 ++ api/app/domain/services/agent_task_runner.py | 1 + .../domain/services/flows/planner_react.py | 2 + .../domain/services/tools/minimax_music.py | 55 ++++++++ .../domain/services/tools/multimodal_core.py | 124 ++++++++++++++++++ api/tests/test_minimax_music_tool.py | 121 +++++++++++++++++ ui/src/components/tool-preview-panel.tsx | 1 + 7 files changed, 314 insertions(+) create mode 100644 api/app/domain/services/tools/minimax_music.py create mode 100644 api/tests/test_minimax_music_tool.py diff --git a/.env.example b/.env.example index dfe9817..b79d5e1 100644 --- a/.env.example +++ b/.env.example @@ -100,6 +100,16 @@ MINIMAX_BASE_URL= # 声音复刻与合成模型 MINIMAX_VOICE_CLONE_MODEL=speech-2.8-hd +# MiniMax 音乐生成配置 🆕 +# 音乐生成模型: music-3.0 / music-2.6 / music-3.0-free / music-2.6-free +MINIMAX_MUSIC_MODEL=music-3.0 +# 接口返回形式: hex（直接返回音频数据）或 url（返回临时下载地址，有效期24小时） +MINIMAX_MUSIC_OUTPUT_FORMAT=hex +# 音频编码格式: mp3 / wav / pcm +MINIMAX_MUSIC_AUDIO_FORMAT=mp3 +# 是否添加AIGC水印（仅 cn_zh 区域支持） +MINIMAX_MUSIC_AIGC_WATERMARK=false + # 人脸检测配置 # 检测方法: opencv（轻量级，使用OpenCV Haar Cascade）或 llm（大模型，使用火山引擎多模态模型） FACE_DETECTION_METHOD=opencv diff --git a/api/app/domain/services/agent_task_runner.py b/api/app/domain/services/agent_task_runner.py index 111109f..3d71d04 100644 --- a/api/app/domain/services/agent_task_runner.py +++ b/api/app/domain/services/agent_task_runner.py @@ -602,6 +602,7 @@ async def _handle_tool_event(self, event: ToolEvent) -> None: "virtual_anchor", "qwen_tts", "minimax_voice", + "minimax_music", "audio_mixing", ]: result_payload: Any = None diff --git a/api/app/domain/services/flows/planner_react.py b/api/app/domain/services/flows/planner_react.py index 840fb6e..cd63068 100644 --- a/api/app/domain/services/flows/planner_react.py +++ b/api/app/domain/services/flows/planner_react.py @@ -21,6 +21,7 @@ from app.domain.services.tools.image_generation import ImageGenerationTool from app.domain.services.tools.mcp import MCPTool from app.domain.services.tools.message import MessageTool +from app.domain.services.tools.minimax_music import MiniMaxMusicTool from app.domain.services.tools.minimax_voice import MiniMaxVoiceTool from app.domain.services.tools.model_3d_generation import Model3DGenerationTool from app.domain.services.tools.qwen_tts import QwenTTSTool @@ -75,6 +76,7 @@ def __init__( VirtualAnchorGenerationTool(), QwenTTSTool(), MiniMaxVoiceTool(), + MiniMaxMusicTool(), AudioMixingTool(), mcp_tool, a2a_tool, diff --git a/api/app/domain/services/tools/minimax_music.py b/api/app/domain/services/tools/minimax_music.py new file mode 100644 index 0000000..eef5c35 --- /dev/null +++ b/api/app/domain/services/tools/minimax_music.py @@ -0,0 +1,55 @@ +from typing import Optional + +from app.domain.models.tool_result import ToolResult +from .base import BaseTool, tool +from .multimodal_core import MultimodalCore + + +class MiniMaxMusicTool(BaseTool): + name: str = "minimax_music" + + def __init__(self) -> None: + super().__init__() + self.core = MultimodalCore() + + @tool( + name="minimax_music_generation", + description="根据风格描述与歌词生成音乐，返回本地音频文件。", + parameters={ + "prompt": {"type": "string", "description": "音乐风格、情绪或场景描述"}, + "lyrics": {"type": "string", "description": "歌词，换行分隔，可选"}, + "model": {"type": "string", "description": "音乐生成模型，可选"}, + "output_format": { + "type": "string", + "enum": list(MultimodalCore.MINIMAX_MUSIC_OUTPUT_FORMATS), + "description": "接口返回形式，可选", + }, + "audio_format": { + "type": "string", + "enum": list(MultimodalCore.MINIMAX_MUSIC_AUDIO_FORMATS), + "description": "音频编码格式，可选", + }, + "is_instrumental": {"type": "boolean", "description": "是否生成纯伴奏，可选"}, + "lyrics_optimizer": {"type": "boolean", "description": "是否自动优化歌词，可选"}, + }, + required=["prompt"], + ) + async def minimax_music_generation( + self, + prompt: Optional[str] = None, + lyrics: Optional[str] = None, + model: Optional[str] = None, + output_format: Optional[str] = None, + audio_format: Optional[str] = None, + is_instrumental: Optional[bool] = None, + lyrics_optimizer: Optional[bool] = None, + ) -> ToolResult: + return await self.core.generate_music( + prompt=prompt, + lyrics=lyrics, + model=model, + output_format=output_format, + audio_format=audio_format, + is_instrumental=is_instrumental, + lyrics_optimizer=lyrics_optimizer, + ) diff --git a/api/app/domain/services/tools/multimodal_core.py b/api/app/domain/services/tools/multimodal_core.py index 3c4907b..8870ed1 100644 --- a/api/app/domain/services/tools/multimodal_core.py +++ b/api/app/domain/services/tools/multimodal_core.py @@ -18,6 +18,13 @@ class MultimodalCore: + # MiniMax music generation constraints + MINIMAX_MUSIC_OUTPUT_FORMATS = ("url", "hex") + MINIMAX_MUSIC_AUDIO_FORMATS = ("mp3", "wav", "pcm") + MINIMAX_MUSIC_STATUS_IN_PROGRESS = 1 + MINIMAX_MUSIC_STATUS_COMPLETED = 2 + MINIMAX_MUSIC_URL_TTL_HOURS = 24 + def __init__(self) -> None: self.base_dir = Path(__file__).resolve().parents[4] self.storage_dir = self.base_dir / "storage" @@ -64,6 +71,10 @@ def __init__(self) -> None: self.minimax_base_url = os.getenv("MINIMAX_BASE_URL", "").strip() self.minimax_voice_clone_model = os.getenv("MINIMAX_VOICE_CLONE_MODEL", "speech-2.8-hd").strip() self.minimax_voice_synthesis_model = os.getenv("MINIMAX_VOICE_SYNTHESIS_MODEL", self.minimax_voice_clone_model).strip() + self.minimax_music_model = os.getenv("MINIMAX_MUSIC_MODEL", "music-3.0").strip() + self.minimax_music_output_format = os.getenv("MINIMAX_MUSIC_OUTPUT_FORMAT", "hex").strip().lower() + self.minimax_music_audio_format = os.getenv("MINIMAX_MUSIC_AUDIO_FORMAT", "mp3").strip().lower() + self.minimax_music_aigc_watermark = os.getenv("MINIMAX_MUSIC_AIGC_WATERMARK", "false").strip().lower() == "true" self.face_detection_method = os.getenv("FACE_DETECTION_METHOD", "llm").strip().lower() @staticmethod @@ -1052,6 +1063,119 @@ async def minimax_voice_cloning(self, reference_audio: str, text: str, model: Op except Exception as e: return ToolResult(success=False, message=f"声音复刻失败: {e}") + def _minimax_music_payload( + self, + model: str, + prompt: Optional[str], + lyrics: Optional[str], + output_format: str, + audio_format: str, + is_instrumental: Optional[bool], + lyrics_optimizer: Optional[bool], + ) -> Dict[str, Any]: + """Build the /v1/music_generation request body; only `model` is mandatory.""" + payload: Dict[str, Any] = { + "model": model, + "output_format": output_format, + # Streamed responses only carry hex chunks, so always request the + # non-streaming form and persist one complete file instead. + "stream": False, + "audio_setting": {"format": audio_format}, + } + if prompt: + payload["prompt"] = prompt + if lyrics: + payload["lyrics"] = lyrics + if is_instrumental is not None: + payload["is_instrumental"] = is_instrumental + if lyrics_optimizer is not None: + payload["lyrics_optimizer"] = lyrics_optimizer + # `aigc_watermark` is only accepted by the cn_zh endpoint + if self.minimax_region == "cn_zh" and self.minimax_music_aigc_watermark: + payload["aigc_watermark"] = True + return payload + + def _extract_minimax_music_audio(self, data: Dict[str, Any]) -> str: + """Validate a music generation response and return the completed audio payload.""" + base_resp = (data.get("base_resp") or {}) if isinstance(data, dict) else {} + if base_resp.get("status_code") not in (0, None): + raise RuntimeError(base_resp.get("status_msg") or f"status_code={base_resp.get('status_code')}") + result = (data.get("data") or {}) if isinstance(data, dict) else {} + status = result.get("status") + if status == self.MINIMAX_MUSIC_STATUS_IN_PROGRESS: + raise RuntimeError("音乐仍在生成中，请稍后重试") + if status not in (self.MINIMAX_MUSIC_STATUS_COMPLETED, None): + raise RuntimeError(f"音乐生成未完成: status={status}") + audio = result.get("audio") + if not audio: + raise RuntimeError("音乐生成返回为空") + return audio + + async def _store_minimax_music_audio(self, audio: str, output_format: str, audio_format: str, text: str) -> str: + """Persist a url- or hex-form music result into the local audio directory.""" + ext = f".{audio_format}" + if output_format == "url": + return await self._download_to(audio, self.audios_dir, "audios", "music", text, ext) + return self._save_bytes_to(bytes.fromhex(audio), self.audios_dir, "audios", "music", text, ext) + + async def generate_music( + self, + prompt: Optional[str] = None, + lyrics: Optional[str] = None, + model: Optional[str] = None, + output_format: Optional[str] = None, + audio_format: Optional[str] = None, + is_instrumental: Optional[bool] = None, + lyrics_optimizer: Optional[bool] = None, + ) -> ToolResult: + try: + if not self.minimax_api_key: + return ToolResult(success=False, message="未配置 MINIMAX_API_KEY") + if not prompt and not lyrics: + return ToolResult(success=False, message="prompt 与 lyrics 至少需要提供一个") + music_model = (model or self.minimax_music_model).strip() + resolved_output = (output_format or self.minimax_music_output_format or "hex").strip().lower() + if resolved_output not in self.MINIMAX_MUSIC_OUTPUT_FORMATS: + return ToolResult(success=False, message=f"不支持的返回格式: {resolved_output}，可选 {'/'.join(self.MINIMAX_MUSIC_OUTPUT_FORMATS)}") + resolved_audio = (audio_format or self.minimax_music_audio_format or "mp3").strip().lower() + if resolved_audio not in self.MINIMAX_MUSIC_AUDIO_FORMATS: + return ToolResult(success=False, message=f"不支持的音频格式: {resolved_audio}，可选 {'/'.join(self.MINIMAX_MUSIC_AUDIO_FORMATS)}") + payload = self._minimax_music_payload( + music_model, + prompt, + lyrics, + resolved_output, + resolved_audio, + is_instrumental, + lyrics_optimizer, + ) + async with httpx.AsyncClient(timeout=300) as client: + resp = await client.post( + self._minimax_endpoint("/v1/music_generation"), + json=payload, + headers=self._minimax_headers(), + ) + resp.raise_for_status() + data = resp.json() + audio = self._extract_minimax_music_audio(data) + audio_url = await self._store_minimax_music_audio(audio, resolved_output, resolved_audio, prompt or lyrics or "") + result_data: Dict[str, Any] = { + "audio_url": audio_url, + "local_path": audio_url, + "model": music_model, + "output_format": resolved_output, + "audio_format": resolved_audio, + "is_instrumental": bool(is_instrumental), + "provider": "minimax-music-generation", + } + if resolved_output == "url": + # The remote address expires, so keep it only for troubleshooting + result_data["original_url"] = audio + result_data["original_url_ttl_hours"] = self.MINIMAX_MUSIC_URL_TTL_HOURS + return ToolResult(success=True, message="音乐生成成功", data=result_data) + except Exception as e: + return ToolResult(success=False, message=f"音乐生成失败: {e}") + async def concatenate_audio(self, audio_files: List[str], crossfade_duration: int = 200, silence_duration: int = 1200) -> ToolResult: try: if len(audio_files) < 2: diff --git a/api/tests/test_minimax_music_tool.py b/api/tests/test_minimax_music_tool.py new file mode 100644 index 0000000..574a9e9 --- /dev/null +++ b/api/tests/test_minimax_music_tool.py @@ -0,0 +1,121 @@ +"""Unit tests for the MiniMax music generation tool wiring.""" +import asyncio + +from app.domain.services.tools.minimax_music import MiniMaxMusicTool + + +def _tool_without_key() -> MiniMaxMusicTool: + tool = MiniMaxMusicTool() + tool.core.minimax_api_key = "" + return tool + + +def _tool_with_key() -> MiniMaxMusicTool: + tool = MiniMaxMusicTool() + tool.core.minimax_api_key = "test-key" + return tool + + +def test_tool_registers_music_generation(): + """The music generation operation is exposed to the LLM tool schema.""" + tool = MiniMaxMusicTool() + names = {schema["function"]["name"] for schema in tool.get_tools()} + assert "minimax_music_generation" in names + assert tool.has_tool("minimax_music_generation") + + +def test_music_endpoint_covers_both_regions(): + """Music generation resolves the global and CN hosts of the same path.""" + core = MiniMaxMusicTool().core + core.minimax_base_url = "" + core.minimax_group_id = "" + core.minimax_region = "global_en" + assert core._minimax_endpoint("/v1/music_generation") == "https://api.minimax.io/v1/music_generation" + core.minimax_region = "cn_zh" + assert core._minimax_endpoint("/v1/music_generation") == "https://api.minimaxi.com/v1/music_generation" + + +def test_payload_defaults_and_optional_fields(): + """Only the model is mandatory; optional fields appear when provided.""" + core = MiniMaxMusicTool().core + core.minimax_region = "global_en" + minimal = core._minimax_music_payload("music-3.0", "calm piano", None, "hex", "mp3", None, None) + assert minimal == { + "model": "music-3.0", + "output_format": "hex", + "stream": False, + "audio_setting": {"format": "mp3"}, + "prompt": "calm piano", + } + full = core._minimax_music_payload("music-2.6", "rock", "line one\nline two", "url", "wav", True, False) + assert full["lyrics"] == "line one\nline two" + assert full["is_instrumental"] is True + assert full["lyrics_optimizer"] is False + assert full["output_format"] == "url" + assert full["audio_setting"] == {"format": "wav"} + + +def test_watermark_field_is_cn_only(): + """The watermark flag is only sent to the CN endpoint.""" + core = MiniMaxMusicTool().core + core.minimax_music_aigc_watermark = True + core.minimax_region = "global_en" + assert "aigc_watermark" not in core._minimax_music_payload("music-3.0", "jazz", None, "hex", "mp3", None, None) + core.minimax_region = "cn_zh" + assert core._minimax_music_payload("music-3.0", "jazz", None, "hex", "mp3", None, None)["aigc_watermark"] is True + + +def test_completed_response_returns_audio(): + """A completed task exposes the audio payload.""" + core = MiniMaxMusicTool().core + data = {"data": {"status": 2, "audio": "1a2b3c"}, "base_resp": {"status_code": 0}} + assert core._extract_minimax_music_audio(data) == "1a2b3c" + + +def test_in_progress_response_is_rejected(): + """An unfinished task does not yield audio.""" + core = MiniMaxMusicTool().core + data = {"data": {"status": 1, "audio": ""}, "base_resp": {"status_code": 0}} + try: + core._extract_minimax_music_audio(data) + except RuntimeError: + pass + else: + raise AssertionError("in-progress response should raise") + + +def test_api_error_and_empty_audio_are_rejected(): + """Non-zero status codes and empty audio are reported as failures.""" + core = MiniMaxMusicTool().core + for data in ( + {"data": {"status": 2, "audio": "1a2b"}, "base_resp": {"status_code": 1004, "status_msg": "bad key"}}, + {"data": {"status": 2, "audio": ""}, "base_resp": {"status_code": 0}}, + ): + try: + core._extract_minimax_music_audio(data) + except RuntimeError: + continue + raise AssertionError("invalid response should raise") + + +def test_music_generation_requires_api_key(): + """Music generation fails gracefully when no API key is configured.""" + tool = _tool_without_key() + result = asyncio.run(tool.core.generate_music(prompt="calm piano")) + assert result.success is False + + +def test_music_generation_requires_prompt_or_lyrics(): + """At least one of prompt or lyrics must be supplied.""" + tool = _tool_with_key() + result = asyncio.run(tool.core.generate_music()) + assert result.success is False + + +def test_music_generation_validates_formats(): + """Unsupported output and audio formats are rejected before any request.""" + tool = _tool_with_key() + bad_output = asyncio.run(tool.core.generate_music(prompt="calm piano", output_format="mp4")) + assert bad_output.success is False + bad_audio = asyncio.run(tool.core.generate_music(prompt="calm piano", audio_format="flac")) + assert bad_audio.success is False diff --git a/ui/src/components/tool-preview-panel.tsx b/ui/src/components/tool-preview-panel.tsx index a90f7bc..b61bcdc 100644 --- a/ui/src/components/tool-preview-panel.tsx +++ b/ui/src/components/tool-preview-panel.tsx @@ -55,6 +55,7 @@ function getCompactToolContent(tool: ToolEvent): unknown { 'virtual_anchor', 'qwen_tts', 'minimax_voice', + 'minimax_music', 'audio_mixing', ]) if (!multimodalToolNames.has(tool.name)) return tool.content