feat(diarization): 支持字幕说话者区分

docs/docs/features/subtitle-generation.md

@@ -57,6 +57,14 @@ keywords:
 - 字幕显示按语音结束时间收敛，减少「字幕滞留」
 - **超长音频（4 小时以上）自动在静音处分段**处理，时间轴无缝合并，不再爆内存
 
+## 角色分离
+
+标准的「转写」和「转写 + 翻译」任务可在高级设置中开启**角色分离**。它会在转写、精修与翻译完成后，本地分析整段音频，把不同角色按时间轴对齐到每条字幕；处理耗时与音频长度大致成正比。
+
+首次使用前需在「资源中心 → 本地多模型引擎」确认运行库可用并下载角色分离模型。角色信息默认只保存在应用内校对数据中，不会改变导出的字幕；需要交付可读标签时，可同时开启「将角色标签写入字幕文件」，输出 `[Speaker 1]` 等前缀。
+
+当前版本暂不在一条龙向导、自定义配方及含配音 / 合成的流程中开放角色分离；这些流程会在角色元数据与配音音色映射完成后再接入。
+
 ## GPU 加速
 
 | 平台                          | 加速后端           |

extraResources/sherpa/worker/speaker-diarization-config.js

@@ -0,0 +1,49 @@
+'use strict';
+
+// 说话者分离配置纯函数。worker 与定向单测共同 require 本文件，避免原生参数
+// 映射出现两份实现。零依赖，输入路径由主进程完成校验后传入。
+
+/**
+ * @typedef {Object} SpeakerDiarizationRequest
+ * @property {string} segmentationModel
+ * @property {string} embeddingModel
+ * @property {number} [numClusters] 正数为已知说话者数；其它值使用自动聚类。
+ * @property {number} [numThreads]
+ */
+
+/**
+ * 构建 sherpa-onnx OfflineSpeakerDiarizationConfig。
+ *
+ * @param {SpeakerDiarizationRequest} req
+ * @returns {object}
+ */
+function buildSpeakerDiarizationConfig(req) {
+  const numThreads = Math.max(1, Math.min(8, Number(req.numThreads) || 2));
+  const numClusters =
+    Number.isInteger(req.numClusters) && req.numClusters > 0
+      ? req.numClusters
+      : -1;
+
+  return {
+    segmentation: {
+      pyannote: { model: req.segmentationModel },
+      numThreads,
+      debug: 0,
+      provider: 'cpu',
+    },
+    embedding: {
+      model: req.embeddingModel,
+      numThreads,
+      debug: 0,
+      provider: 'cpu',
+    },
+    clustering: {
+      numClusters,
+      threshold: 0.5,
+    },
+    minDurationOn: 0.2,
+    minDurationOff: 0.5,
+  };
+}
+
+module.exports = { buildSpeakerDiarizationConfig };

extraResources/sherpa/worker/speaker-diarization-worker.js

@@ -0,0 +1,56 @@
+'use strict';
+// 独立说话者分离 worker：加载 pyannote segmentation + 3D-Speaker embedding，
+// 读取 16k WAV 后执行离线 diarization。原生推理完全隔离在 utilityProcess，
+// native 异常不会带崩 Electron 主进程；主进程取消时直接终止本进程。
+const path = require('path');
+const {
+  buildSpeakerDiarizationConfig,
+} = require('./speaker-diarization-config.js');
+
+const channel = (() => {
+  if (process.parentPort) {
+    return {
+      post: (msg) => process.parentPort.postMessage(msg),
+      onMessage: (cb) => process.parentPort.on('message', (e) => cb(e.data)),
+    };
+  }
+  const { parentPort } = require('worker_threads');
+  return {
+    post: (msg) => parentPort.postMessage(msg),
+    onMessage: (cb) => parentPort.on('message', cb),
+  };
+})();
+
+const sherpa = require(path.join(__dirname, '..', 'vendor', 'sherpa-onnx.js'));
+
+function diarize(req) {
+  const diarizer = new sherpa.OfflineSpeakerDiarization(
+    buildSpeakerDiarizationConfig(req),
+  );
+  const wave = sherpa.readWave(req.audioFile, false);
+  if (wave.sampleRate !== diarizer.sampleRate) {
+    throw new Error(
+      `speaker diarization expects ${diarizer.sampleRate} Hz audio, got ${wave.sampleRate} Hz`,
+    );
+  }
+  const raw = diarizer.process(wave.samples) || [];
+  const segments = raw.map((segment) => ({
+    start: Number(segment.start),
+    end: Number(segment.end),
+    speaker: Number(segment.speaker),
+  }));
+  channel.post({ type: 'done', id: req.id, segments });
+}
+
+channel.onMessage((req) => {
+  if (req.type !== 'diarize') return;
+  try {
+    diarize(req);
+  } catch (error) {
+    channel.post({
+      type: 'error',
+      id: req.id,
+      message: error instanceof Error ? error.message : String(error),
+    });
+  }
+});

main/helpers/dubbing/dubbingProcessor.ts

@@ -20,6 +20,7 @@ import {
   serializeSubtitleCues,
   type SubtitleCue,
 } from '../subtitleFormats';
+import { normalizeDubbingSpeechText } from './textNormalization';
 import {
   computeSlots,
   estimateDurationMs,
@@ -525,8 +526,10 @@ async function synthesizeAndAlignCue(
       : 1;
   const voiceId = cue.voiceId || config.voice;
   const wavPath = path.join(session.workDir, `cue-${cue.index}.wav`);
+  // 会话/展示字幕保留 `[Speaker N]`；仅在估时与 TTS 的最终输入边界剥离。
+  const speechText = normalizeDubbingSpeechText(cue.text);
 
-  if (!cue.text) {
+  if (!speechText) {
     // 空行：静音占位，无需合成。
     cue.status = 'done';
     cue.finalMs = 0;
@@ -538,7 +541,7 @@ async function synthesizeAndAlignCue(
 
   // 第 1 层：预估（含校准与整体语速）→ speed 预控制。
   const est = calibratedEstimate(
-    estimateDurationMs(cue.text),
+    estimateDurationMs(speechText),
     session.calibration,
   );
   const estAtGlobal = Math.round(est / globalSpeed);
@@ -551,7 +554,7 @@ async function synthesizeAndAlignCue(
   let appliedExtra = decision.preSpeed;
   let synthSpeed = globalSpeed * decision.preSpeed;
   let r = await adapter.synthesize(
-    cue.text,
+    speechText,
     voiceId,
     synthSpeed,
     wavPath,
@@ -592,7 +595,7 @@ async function synthesizeAndAlignCue(
     if (recheck.type === 'resynthesize') {
       synthSpeed = globalSpeed * recheck.speed;
       r = await adapter.synthesize(
-        cue.text,
+        speechText,
         voiceId,
         synthSpeed,
         wavPath,

main/helpers/dubbing/textNormalization.ts

@@ -0,0 +1,12 @@
+import { stripSpeakerLabelPrefix } from '../speakerDiarization/alignment';
+
+/**
+ * 配音引擎实际朗读的文本规范化。
+ *
+ * 说话者标签属于字幕展示元数据，不应进入 TTS；其余文本仍沿用原有的换行折叠与
+ * 首尾空白清理行为。
+ */
+export function normalizeDubbingSpeechText(text: string): string {
+  const flattened = (text || '').replace(/\n+/g, ' ').trim();
+  return stripSpeakerLabelPrefix(flattened).trim();
+}

main/helpers/fileProcessor.ts

@@ -44,6 +44,13 @@ import {
   TaskCancelledError,
   getTaskContext,
 } from './taskContext';
+import { runSpeakerDiarizationStage } from './speakerDiarization/stage';
+import type { SpeakerDiarizationSegment } from './speakerDiarization/alignment';
+import {
+  isSpeakerDiarizationStandardTaskContext,
+  shouldExtractAudioForEmbeddedSubtitle,
+  getSpeakerDiarizationMetadataWarning,
+} from '../../types/speakerDiarization';
 
 /**
  * 处理任务错误
@@ -271,18 +278,21 @@ export async function processFile(
     'prepareSubtitle',
     'refineSubtitle',
     'translateSubtitle',
+    'speakerDiarization',
     'dubbing',
     'composeVideo',
     'extractAudioProgress',
     'extractSubtitleProgress',
     'refineSubtitleProgress',
     'translateSubtitleProgress',
+    'speakerDiarizationProgress',
     'dubbingProgress',
     'composeVideoProgress',
     'extractAudioError',
     'extractSubtitleError',
     'refineSubtitleError',
     'translateSubtitleError',
+    'speakerDiarizationError',
     'dubbingError',
     'composeVideoError',
   ]) {
@@ -321,6 +331,13 @@ export async function processFile(
     const translationActive =
       shouldTranslateSubtitle && translateProvider !== '-1';
 
+    const speakerDiarizationStageActive =
+      !isSubtitleFile &&
+      shouldGenerateSubtitle &&
+      !hasProvidedSubtitle &&
+      formData?.speakerDiarization === true &&
+      isSpeakerDiarizationStandardTaskContext(formData);
+
     /** 文件停靠在人工检查点：置待校对、发聚合通知、结束本轮（不占并发槽） */
     const dockAtGate = (gate: 'subtitle' | 'dubbing') => {
       const field = gate === 'subtitle' ? 'subtitleGate' : 'dubbingGate';
@@ -482,6 +499,22 @@ export async function processFile(
             if (!srtHasCues(srtContent)) {
               throw new Error('extracted embedded subtitle has no cues');
             }
+            // 内封字幕只替代 ASR，不替代角色分离所需的整段音频。
+            // 在角色分离开启时仍抽取并记录 tempAudioFile，供后处理阶段使用。
+            if (shouldExtractAudioForEmbeddedSubtitle(formData)) {
+              logMessage(
+                `extract audio for speaker diarization: ${fileName}`,
+                'info',
+              );
+              throwIfTaskCancelled();
+              const tempAudioFile = await extractAudioFromVideo(event, file);
+              if (saveAudio) {
+                const audioFileName = `${fileName}.wav`;
+                const targetAudioPath = path.join(directory, audioFileName);
+                file.audioFile = targetAudioPath;
+                fs.copyFileSync(tempAudioFile, targetAudioPath);
+              }
+            }
             event.sender.send('taskFileChange', {
               ...file,
               extractAudio: 'done',
@@ -667,8 +700,32 @@ export async function processFile(
       await stripSourceSubtitlePunctuation(file.srtFile, fileName);
     }
 
+    // 可选角色分离：仅对标准字幕任务中本轮真实 ASR 的音频执行。放在翻译之后，
+    // 避免角色信息污染翻译提示；独立阶段保持 loading，直到 sidecar 写入完成后
+    // 才置 done，从而保证校对入口不会抢先读到旧内容。
+    let speakerSegments: SpeakerDiarizationSegment[] | undefined;
+    let speakerDiarizationWarning: string | undefined;
+    if (speakerDiarizationStageActive) {
+      throwIfTaskCancelled();
+      file.speakerDiarization = 'loading';
+      file.speakerDiarizationProgress = 0;
+      delete file.speakerDiarizationError;
+      event.sender.send('taskFileChange', { ...file });
+      logMessage(`speaker diarization stage started: ${fileName}`, 'info');
+      const result = await runSpeakerDiarizationStage({
+        file,
+        formData,
+        signal: getTaskContext()?.signal,
+      });
+      speakerSegments = result.segments;
+      speakerDiarizationWarning = result.reason;
+    }
+
+    throwIfTaskCancelled();
+    let speakerMetadataPersisted = false;
+    let proofreadDataFailure: string | undefined;
     if (file.srtFile && fs.existsSync(file.srtFile)) {
-      const proofreadDataFile = await writeProofreadDataFromFiles({
+      const proofreadDataResult = await writeProofreadDataFromFiles({
         file,
         sourceFile: file.srtFile,
         targetFile:
@@ -683,13 +740,43 @@ export async function processFile(
         targetLanguage,
         translateContent: formData?.translateContent,
         outputFormat: formData?.subtitleOutputFormat,
+        speakerSegments,
       });
-      if (proofreadDataFile) {
-        file.proofreadDataFile = proofreadDataFile;
+      if ('filePath' in proofreadDataResult) {
+        file.proofreadDataFile = proofreadDataResult.filePath;
+        speakerMetadataPersisted = true;
         event.sender.send('taskFileChange', file);
+      } else {
+        proofreadDataFailure = `${proofreadDataResult.reason}${proofreadDataResult.error ? `: ${proofreadDataResult.error}` : ''}`;
       }
     }
 
+    const metadataWarning = getSpeakerDiarizationMetadataWarning(
+      Boolean(speakerSegments?.length),
+      speakerMetadataPersisted,
+    );
+    if (speakerDiarizationStageActive && metadataWarning) {
+      file.speakerDiarizationError = metadataWarning;
+      speakerDiarizationWarning = 'metadata-save-failed';
+      logMessage(
+        `speaker diarization metadata could not be saved${proofreadDataFailure ? ` (${proofreadDataFailure})` : ''}: ${fileName}`,
+        'warning',
+      );
+    }
+
+    if (speakerDiarizationStageActive) {
+      throwIfTaskCancelled();
+      file.speakerDiarization = 'done';
+      file.speakerDiarizationProgress = 100;
+      event.sender.send('taskFileChange', { ...file });
+      logMessage(
+        speakerDiarizationWarning
+          ? `speaker diarization stage done with warning (${speakerDiarizationWarning}): ${fileName}`
+          : `speaker diarization stage done: ${fileName}`,
+        speakerDiarizationWarning ? 'warning' : 'info',
+      );
+    }
+
     // 将交付字幕转换为用户选择的输出格式（内部流程始终为 SRT，此处仅转换最终交付物）
     const outputFormat = resolveOutputFormat(formData);
     if (outputFormat !== 'srt') {
@@ -772,6 +859,7 @@ export async function processFile(
         extractAudio: '',
         extractSubtitle: '',
         translateSubtitle: '',
+        speakerDiarization: '',
       });
       return;
     }

main/helpers/ipcRecipeHandlers.ts

@@ -7,10 +7,18 @@ import { ipcMain } from 'electron';
 import { randomUUID } from 'crypto';
 import { store, logMessage } from './storeManager';
 import type { TaskRecipe } from '../../types/recipe';
+import { stripSpeakerDiarizationConfig } from '../../types/speakerDiarization';
 
 function readRecipes(): TaskRecipe[] {
   const list = store.get('taskRecipes');
-  return Array.isArray(list) ? list : [];
+  return Array.isArray(list)
+    ? list.map((recipe) => ({
+        ...recipe,
+        config: recipe.config
+          ? stripSpeakerDiarizationConfig(recipe.config)
+          : undefined,
+      }))
+    : [];
 }
 
 export function setupRecipeHandlers(): void {
@@ -25,7 +33,9 @@ export function setupRecipeHandlers(): void {
       name,
       goals: recipe.goals,
       accepts: recipe.accepts,
-      config: recipe.config,
+      config: recipe.config
+        ? stripSpeakerDiarizationConfig(recipe.config)
+        : undefined,
       createdAt: Date.now(),
     };
     const index = list.findIndex((r) => r.id === saved.id);