feat(diarization): Phase 3 stop attribution — mic timeline is 'You', diarize far side masked (FR-SPK)
stop_recording now, when the mic is on, takes 'You' spans straight from the MicActivity timeline and diarizes the recording with those ranges zeroed, so sherpa only ever clusters the far side into Speaker N. Merges the span lists and names them via the existing voiceprint map (You + Speaker 2..). Falls back to the whole-signal pass + voiceprint match when the mic is off. Adds mask_ranges helper + tests.
This commit is contained in:
+136
-52
@@ -242,6 +242,25 @@ fn diarizer_from_installed_models() -> Option<SherpaDiarizer> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Zero out `ranges` (given as `(start_ms, end_ms)`) in 16kHz mono `samples` —
|
||||
/// used to remove the mic's "You" ranges from the recording before diarizing the
|
||||
/// far side, so clustering never sees the mic (Phase 3, FR-SPK).
|
||||
fn mask_ranges(mut samples: Vec<f32>, ranges: &[(u64, u64)]) -> Vec<f32> {
|
||||
const SAMPLES_PER_MS: usize = 16; // 16kHz mono
|
||||
for &(start_ms, end_ms) in ranges {
|
||||
let start = (start_ms as usize)
|
||||
.saturating_mul(SAMPLES_PER_MS)
|
||||
.min(samples.len());
|
||||
let end = (end_ms as usize)
|
||||
.saturating_mul(SAMPLES_PER_MS)
|
||||
.min(samples.len());
|
||||
if start < end {
|
||||
samples[start..end].fill(0.0);
|
||||
}
|
||||
}
|
||||
samples
|
||||
}
|
||||
|
||||
/// The distinct speakers seen in `segments` so far, in first-appearance order,
|
||||
/// with any display names applied (T4.3/T4.4, FR-SPK-2/5). Falls back to the
|
||||
/// single pre-diarization "S1" placeholder if no segments exist yet.
|
||||
@@ -339,7 +358,7 @@ pub async fn start_recording(
|
||||
// microphone that fails to open must not sink the meeting: we log and fall
|
||||
// back to loopback-only (the mixer forwards loopback alone once its sink
|
||||
// drops).
|
||||
let (capture, mic_capture, mic_voice_sample) = if settings.microphone_enabled {
|
||||
let (capture, mic_capture, mic_voice_sample, mic_activity) = if settings.microphone_enabled {
|
||||
// The mixer sums both streams for the live transcript; the bridge carries
|
||||
// the mic into the loopback thread so the recorded WAV holds both sides
|
||||
// at native quality (FR-CAP-7).
|
||||
@@ -347,6 +366,9 @@ pub async fn start_recording(
|
||||
// A few seconds of raw mic audio for the post-stop voiceprint match
|
||||
// (bug: mic speaker mislabeled "S1"/"S2" instead of "You").
|
||||
let voice_sample = crate::audio::VoiceSample::new(MIC_VOICEPRINT_SAMPLES);
|
||||
// Phase 3 (FR-SPK): timeline of when the mic ("You") is speaking, filled
|
||||
// by the loopback writer where the mic is folded into audio.wav.
|
||||
let mic_activity = crate::audio::MicActivity::shared();
|
||||
let (loop_sink, mic_sink) = crate::audio::spawn_mixer(frame_tx);
|
||||
let capture = WasapiCapture
|
||||
.start_loopback_recording(
|
||||
@@ -355,6 +377,7 @@ pub async fn start_recording(
|
||||
loop_sink,
|
||||
event_tx.clone(),
|
||||
bridge.clone(),
|
||||
Some(mic_activity.clone()),
|
||||
)
|
||||
.map_err(|e| WaError::new("audio", e.to_string()))?;
|
||||
let mic = WasapiCapture
|
||||
@@ -368,7 +391,8 @@ pub async fn start_recording(
|
||||
.map_err(|e| tracing::warn!("microphone capture unavailable: {e}"))
|
||||
.ok();
|
||||
let voice_sample = mic.is_some().then_some(voice_sample);
|
||||
(capture, mic, voice_sample)
|
||||
let mic_activity = mic.is_some().then_some(mic_activity);
|
||||
(capture, mic, voice_sample, mic_activity)
|
||||
} else {
|
||||
let capture = WasapiCapture
|
||||
.start(
|
||||
@@ -378,7 +402,7 @@ pub async fn start_recording(
|
||||
event_tx,
|
||||
)
|
||||
.map_err(|e| WaError::new("audio", e.to_string()))?;
|
||||
(capture, None, None)
|
||||
(capture, None, None, None)
|
||||
};
|
||||
|
||||
// Fire-and-forget: exits on its own once `event_tx` drops at capture stop;
|
||||
@@ -625,6 +649,7 @@ pub async fn start_recording(
|
||||
diarizer,
|
||||
speaker_names,
|
||||
mic_voice_sample,
|
||||
mic_activity,
|
||||
manual_notes: Arc::new(StdMutex::new(ManualNotes::default())),
|
||||
});
|
||||
drop(guard);
|
||||
@@ -682,67 +707,108 @@ pub async fn stop_recording(
|
||||
.unwrap_or_default();
|
||||
let segment_count = segments.len();
|
||||
|
||||
// T4.1/4.2: one authoritative diarization pass over the now-complete
|
||||
// recording (ADR-0005's "post-stop pass"), refining whatever the live
|
||||
// provisional passes (T4.3) produced. Skipped if diarization models
|
||||
// aren't installed — `speaker_infos_from_segments` then falls back to
|
||||
// the single pre-diarization "S1" placeholder, same as before Phase 4.
|
||||
let mut final_spans: Option<Vec<SpeakerSpan>> = None;
|
||||
if let Some(diarizer) = session.diarizer.clone() {
|
||||
let diarizer_for_task = diarizer.clone();
|
||||
// Speaker attribution over the now-complete recording.
|
||||
// Phase 3 (FR-SPK): with the mic on, the mic *is* "You" — take "You" spans
|
||||
// straight from the activity timeline (no clustering) and diarize the far
|
||||
// side with those ranges masked out, so the mic can never merge into a
|
||||
// speaker cluster. Falls back to the whole-signal pass (ADR-0005) +
|
||||
// voiceprint "You" when there's no mic timeline (mic off) or the pass fails.
|
||||
let mut speaker_names: HashMap<String, String> = HashMap::new();
|
||||
let mut attributed = false;
|
||||
if let (Some(diarizer), Some(mic_activity)) =
|
||||
(session.diarizer.clone(), session.mic_activity.clone())
|
||||
{
|
||||
let you_ms = mic_activity.you_spans();
|
||||
let wav_path = session.wav_path.clone();
|
||||
match tauri::async_runtime::spawn_blocking(move || diarizer_for_task.diarize(&wav_path))
|
||||
.await
|
||||
{
|
||||
Ok(Ok(spans)) => {
|
||||
let diarizer_for_task = diarizer.clone();
|
||||
let you_for_task = you_ms.clone();
|
||||
let far = tauri::async_runtime::spawn_blocking(move || {
|
||||
let samples =
|
||||
crate::audio::read_wav_mono_16k(&wav_path).map_err(|e| e.to_string())?;
|
||||
diarizer_for_task
|
||||
.diarize_samples(mask_ranges(samples, &you_for_task))
|
||||
.map_err(|e| e.to_string())
|
||||
})
|
||||
.await;
|
||||
match far {
|
||||
Ok(Ok(far_spans)) => {
|
||||
let mut spans: Vec<SpeakerSpan> = you_ms
|
||||
.iter()
|
||||
.map(|&(start_ms, end_ms)| SpeakerSpan {
|
||||
start_ms,
|
||||
end_ms,
|
||||
speaker: "You".to_string(),
|
||||
})
|
||||
.collect();
|
||||
spans.extend(far_spans);
|
||||
spans.sort_by_key(|s| s.start_ms);
|
||||
diarizer.assign(&mut segments, &spans);
|
||||
final_spans = Some(spans);
|
||||
// Reuse the voiceprint path's naming: "You" -> "You", every far
|
||||
// speaker in first-appearance order -> "Speaker 2", "Speaker 3"….
|
||||
let labels = crate::diarization::voiceprint::first_appearance_order(&spans);
|
||||
speaker_names = crate::diarization::voiceprint::build_name_map(&labels, "You");
|
||||
attributed = true;
|
||||
}
|
||||
Ok(Err(e)) => tracing::warn!("final diarization pass failed: {e}"),
|
||||
Err(e) => tracing::warn!("final diarization task failed: {e}"),
|
||||
Ok(Err(e)) => tracing::warn!("phase 3 far-side diarization failed: {e}"),
|
||||
Err(e) => tracing::warn!("phase 3 diarization task failed: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
// Bug fix: identify which diarized cluster is the mic (voiceprint match
|
||||
// against the mic-only sample) and auto-label it "You" — otherwise the
|
||||
// mic speaker is just whichever cluster sherpa-onnx happened to call
|
||||
// "S1". Never overrides a name the user already set live (T4.4).
|
||||
if let (Some(voice_sample), Some(spans)) = (&session.mic_voice_sample, &final_spans) {
|
||||
let mic_samples = voice_sample.samples();
|
||||
match crate::diarization::voiceprint::match_mic_speaker(
|
||||
&diarization_embedding_model_file(),
|
||||
&mic_samples,
|
||||
&session.wav_path,
|
||||
spans,
|
||||
) {
|
||||
Ok(auto_names) => {
|
||||
for (label, name) in auto_names {
|
||||
let already_named = session
|
||||
.speaker_names
|
||||
.lock()
|
||||
.map(|g| g.contains_key(&label))
|
||||
.unwrap_or(true); // poisoned lock: don't guess, skip
|
||||
if already_named {
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = state.store.rename_speaker(&meeting_id, &label, &name).await {
|
||||
tracing::warn!("failed to persist auto speaker name: {e}");
|
||||
continue;
|
||||
}
|
||||
if let Ok(mut names) = session.speaker_names.lock() {
|
||||
names.insert(label, name);
|
||||
if !attributed {
|
||||
// Fallback: one whole-signal pass, then voiceprint-match the mic against
|
||||
// the clusters to auto-label "You" (never overriding a user-set name).
|
||||
let mut final_spans: Option<Vec<SpeakerSpan>> = None;
|
||||
if let Some(diarizer) = session.diarizer.clone() {
|
||||
let diarizer_for_task = diarizer.clone();
|
||||
let wav_path = session.wav_path.clone();
|
||||
match tauri::async_runtime::spawn_blocking(move || diarizer_for_task.diarize(&wav_path))
|
||||
.await
|
||||
{
|
||||
Ok(Ok(spans)) => {
|
||||
diarizer.assign(&mut segments, &spans);
|
||||
final_spans = Some(spans);
|
||||
}
|
||||
Ok(Err(e)) => tracing::warn!("final diarization pass failed: {e}"),
|
||||
Err(e) => tracing::warn!("final diarization task failed: {e}"),
|
||||
}
|
||||
}
|
||||
if let (Some(voice_sample), Some(spans)) = (&session.mic_voice_sample, &final_spans) {
|
||||
let mic_samples = voice_sample.samples();
|
||||
match crate::diarization::voiceprint::match_mic_speaker(
|
||||
&diarization_embedding_model_file(),
|
||||
&mic_samples,
|
||||
&session.wav_path,
|
||||
spans,
|
||||
) {
|
||||
Ok(auto_names) => {
|
||||
for (label, name) in auto_names {
|
||||
let already_named = session
|
||||
.speaker_names
|
||||
.lock()
|
||||
.map(|g| g.contains_key(&label))
|
||||
.unwrap_or(true); // poisoned lock: don't guess, skip
|
||||
if already_named {
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = state.store.rename_speaker(&meeting_id, &label, &name).await {
|
||||
tracing::warn!("failed to persist auto speaker name: {e}");
|
||||
continue;
|
||||
}
|
||||
if let Ok(mut names) = session.speaker_names.lock() {
|
||||
names.insert(label, name);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!("mic voiceprint match failed: {e}"),
|
||||
}
|
||||
Err(e) => tracing::warn!("mic voiceprint match failed: {e}"),
|
||||
}
|
||||
speaker_names = session
|
||||
.speaker_names
|
||||
.lock()
|
||||
.map(|g| g.clone())
|
||||
.unwrap_or_default();
|
||||
}
|
||||
|
||||
let speaker_names = session
|
||||
.speaker_names
|
||||
.lock()
|
||||
.map(|g| g.clone())
|
||||
.unwrap_or_default();
|
||||
let speakers = speaker_infos_from_segments(&segments, &speaker_names);
|
||||
let backend_used = session
|
||||
.active_backend
|
||||
@@ -4813,6 +4879,24 @@ mod tests {
|
||||
assert_eq!(labels, vec!["S2", "S1"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_ranges_zeros_only_the_given_ms_ranges() {
|
||||
// 1000ms of 16kHz mono = 16_000 samples, all 1.0.
|
||||
let samples = vec![1.0f32; 16_000];
|
||||
// Mask 100..200ms -> samples [1600, 3200).
|
||||
let out = mask_ranges(samples, &[(100, 200)]);
|
||||
assert_eq!(out[1599], 1.0); // just before the range
|
||||
assert_eq!(out[1600], 0.0); // range start zeroed
|
||||
assert_eq!(out[3199], 0.0); // range end (exclusive) zeroed
|
||||
assert_eq!(out[3200], 1.0); // just after the range
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_ranges_clamps_out_of_bounds_ranges() {
|
||||
let out = mask_ranges(vec![1.0f32; 100], &[(0, 10_000)]); // way past the end
|
||||
assert!(out.iter().all(|&s| s == 0.0)); // clamped to len, no panic
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speaker_infos_applies_display_names_by_label() {
|
||||
let segments = vec![segment("S1"), segment("S2")];
|
||||
|
||||
Reference in New Issue
Block a user