From 5bd76b80e816461a9e291a7e84c2aba71956df50 Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Tue, 14 Jul 2026 12:16:03 -0500 Subject: [PATCH] feat(diarization): channel-based split attribution; retire masked-mono path (FR-SPK) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stop + reprocess now, for a split recording, diarize the right (loopback) channel into Speaker N and take You from left-channel (mic) VAD — read back from the file so both agree. Fixes reprocess collapsing to one speaker. Summed recordings keep the whole-signal + voiceprint fallback. Removes MicTimelineFile, phase3_attribute, mask_ranges, mic_activity.json (retirement note retained). --- src-tauri/src/commands.rs | 187 +++++++++++++------------------------- 1 file changed, 61 insertions(+), 126 deletions(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 0218f34..8de9a5a 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -242,44 +242,47 @@ fn diarizer_from_installed_models() -> Option { } } -/// On-disk `mic_activity.json`: the mic's "You" ranges (Phase 3, FR-SPK), -/// persisted next to a retained `audio.wav` so `reprocess_transcript` can -/// re-attribute "You" without the live capture timeline. -#[derive(serde::Serialize, serde::Deserialize)] -struct MicTimelineFile { - schema: u32, - you_spans: Vec<(u64, u64)>, -} +// RETIRED (2026-07-14): the Phase 3 masked-mono attribution (`MicActivity` +// timeline + `mic_activity.json` + `mask_ranges` + `phase3_attribute`). +// Why it existed: to give per-stream "You" vs "Speaker" attribution when +// `audio.wav` was a summed mono mix — record a live mic-speech timeline, then +// diarize the mix with the mic's ranges masked out so clustering saw only the +// far side. Why retired: it was fragile on reprocess (re-aligning a sidecar +// timeline against a mono mix; collapsed to a single speaker), and the +// dual-channel split layout below makes the mic/far-side separation intrinsic +// to the file instead. Commits: 185489f / fd9311c / 0b5a85f (add), and this +// change (remove). Revisit there if a summed-only capture path is ever needed. -/// Phase 3 (FR-SPK) attribution, shared by `stop_recording` and -/// `reprocess_transcript`: label the mic's `you_spans` "You", diarize the -/// recording with those ranges masked out (so clustering only ever sees the far -/// side), assign the merged spans to `segments`, and return the You + Speaker -/// name map. `None` when the far-side pass fails — the caller then falls back to -/// the whole-signal pass + voiceprint match. -async fn phase3_attribute( +/// Split attribution (FR-SPK, supersedes `phase3_attribute`): diarize the far +/// side (right/loopback channel) into `Speaker N`, take "You" straight from +/// left-channel (mic) voice activity, merge + assign + name. Shared by +/// `stop_recording` and `reprocess_transcript`; both read it back from the file +/// so they agree. `None` if the far-side pass fails (caller falls back). +async fn attribute_split( diarizer: Arc, wav_path: PathBuf, - you_spans: Vec<(u64, u64)>, segments: &mut [TranscriptSegment], ) -> Option> { let diarizer_for_task = diarizer.clone(); - let you_for_task = you_spans.clone(); - let far = tauri::async_runtime::spawn_blocking(move || { - let samples = crate::audio::read_wav_mono_16k(&wav_path).map_err(|e| e.to_string())?; - diarizer_for_task - .diarize_samples(mask_ranges(samples, &you_for_task)) - .map_err(|e| e.to_string()) + let result = tauri::async_runtime::spawn_blocking(move || { + // Right channel = loopback/far side; diarize it alone (mic never in it). + let far = crate::audio::read_wav_channel_16k(&wav_path, 1).map_err(|e| e.to_string())?; + let far_spans = diarizer_for_task + .diarize_samples(far) + .map_err(|e| e.to_string())?; + // Left channel = mic; its voice activity is "You". + let mic = crate::audio::read_wav_channel_16k(&wav_path, 0).map_err(|e| e.to_string())?; + Ok::<_, String>((far_spans, crate::audio::vad_spans(&mic))) }) .await; - let far_spans = match far { - Ok(Ok(spans)) => spans, + let (far_spans, you_spans) = match result { + Ok(Ok(v)) => v, Ok(Err(e)) => { - tracing::warn!("phase 3 far-side diarization failed: {e}"); + tracing::warn!("split attribution failed: {e}"); return None; } Err(e) => { - tracing::warn!("phase 3 diarization task failed: {e}"); + tracing::warn!("split attribution task failed: {e}"); return None; } }; @@ -294,31 +297,11 @@ async fn phase3_attribute( spans.extend(far_spans); spans.sort_by_key(|s| s.start_ms); diarizer.assign(segments, &spans); - // Reuse the voiceprint path's naming so labels are uniform app-wide: - // "You" -> "You", every far speaker in first-appearance order -> "Speaker 2"…. + // Uniform naming: "You" -> "You", far speakers -> "Speaker 2", "Speaker 3"…. let labels = crate::diarization::voiceprint::first_appearance_order(&spans); Some(crate::diarization::voiceprint::build_name_map(&labels, "You")) } -/// Zero out `ranges` (given as `(start_ms, end_ms)`) in 16kHz mono `samples` — -/// used to remove the mic's "You" ranges from the recording before diarizing the -/// far side, so clustering never sees the mic (Phase 3, FR-SPK). -fn mask_ranges(mut samples: Vec, ranges: &[(u64, u64)]) -> Vec { - const SAMPLES_PER_MS: usize = 16; // 16kHz mono - for &(start_ms, end_ms) in ranges { - let start = (start_ms as usize) - .saturating_mul(SAMPLES_PER_MS) - .min(samples.len()); - let end = (end_ms as usize) - .saturating_mul(SAMPLES_PER_MS) - .min(samples.len()); - if start < end { - samples[start..end].fill(0.0); - } - } - samples -} - /// The distinct speakers seen in `segments` so far, in first-appearance order, /// with any display names applied (T4.3/T4.4, FR-SPK-2/5). Falls back to the /// single pre-diarization "S1" placeholder if no segments exist yet. @@ -416,17 +399,14 @@ pub async fn start_recording( // microphone that fails to open must not sink the meeting: we log and fall // back to loopback-only (the mixer forwards loopback alone once its sink // drops). - let (capture, mic_capture, mic_voice_sample, mic_activity) = if settings.microphone_enabled { + let (capture, mic_capture, mic_voice_sample) = if settings.microphone_enabled { // The mixer sums both streams for the live transcript; the bridge carries - // the mic into the loopback thread so the recorded WAV holds both sides - // at native quality (FR-CAP-7). + // the mic into the loopback thread. The loopback writes stereo split + // (mic-left/loopback-right) so the two sides stay separate (FR-SPK). let bridge = crate::audio::MicBridge::shared(); - // A few seconds of raw mic audio for the post-stop voiceprint match - // (bug: mic speaker mislabeled "S1"/"S2" instead of "You"). + // A few seconds of raw mic audio for the summed-recording voiceprint + // fallback (imports / mic-off diarization). let voice_sample = crate::audio::VoiceSample::new(MIC_VOICEPRINT_SAMPLES); - // Phase 3 (FR-SPK): timeline of when the mic ("You") is speaking, filled - // by the loopback writer where the mic is folded into audio.wav. - let mic_activity = crate::audio::MicActivity::shared(); let (loop_sink, mic_sink) = crate::audio::spawn_mixer(frame_tx); let capture = WasapiCapture .start_loopback_recording( @@ -435,7 +415,6 @@ pub async fn start_recording( loop_sink, event_tx.clone(), bridge.clone(), - Some(mic_activity.clone()), ) .map_err(|e| WaError::new("audio", e.to_string()))?; let mic = WasapiCapture @@ -449,8 +428,7 @@ pub async fn start_recording( .map_err(|e| tracing::warn!("microphone capture unavailable: {e}")) .ok(); let voice_sample = mic.is_some().then_some(voice_sample); - let mic_activity = mic.is_some().then_some(mic_activity); - (capture, mic, voice_sample, mic_activity) + (capture, mic, voice_sample) } else { let capture = WasapiCapture .start( @@ -460,7 +438,7 @@ pub async fn start_recording( event_tx, ) .map_err(|e| WaError::new("audio", e.to_string()))?; - (capture, None, None, None) + (capture, None, None) }; // Fire-and-forget: exits on its own once `event_tx` drops at capture stop; @@ -707,7 +685,6 @@ pub async fn start_recording( diarizer, speaker_names, mic_voice_sample, - mic_activity, // Mic on → the loopback writer records split (mic-left/loopback-right); // mic off → the summed loopback-only file (FR-SPK). audio_layout: if settings.microphone_enabled { @@ -773,22 +750,21 @@ pub async fn stop_recording( let segment_count = segments.len(); // Speaker attribution over the now-complete recording. - // Phase 3 (FR-SPK): with the mic on, the mic *is* "You" — take "You" spans - // straight from the activity timeline (no clustering) and diarize the far - // side with those ranges masked out, so the mic can never merge into a - // speaker cluster. Falls back to the whole-signal pass (ADR-0005) + - // voiceprint "You" when there's no mic timeline (mic off) or the pass fails. + // FR-SPK: a split recording keeps the mic and far side on separate channels, + // so diarize the far side (right channel) alone → "Speaker N" and take "You" + // from left-channel voice activity. Falls back to the whole-signal pass + // (ADR-0005) + voiceprint "You" for summed recordings (mic off) or if the + // split pass fails. let mut speaker_names: HashMap = HashMap::new(); let mut attributed = false; - if let (Some(diarizer), Some(mic_activity)) = - (session.diarizer.clone(), session.mic_activity.clone()) - { - let you = mic_activity.you_spans(); - if let Some(names) = - phase3_attribute(diarizer, session.wav_path.clone(), you, &mut segments).await - { - speaker_names = names; - attributed = true; + if session.audio_layout == "split" { + if let Some(diarizer) = session.diarizer.clone() { + if let Some(names) = + attribute_split(diarizer, session.wav_path.clone(), &mut segments).await + { + speaker_names = names; + attributed = true; + } } } @@ -907,14 +883,11 @@ pub async fn stop_recording( // ADR-0009: delete the working WAV only after the transcript is finalized // above, and only when retention is off. let voiceprint_path = meeting_dir(&meeting_id).join("voiceprint.wav"); - let mic_activity_path = meeting_dir(&meeting_id).join("mic_activity.json"); if !session.retention { let _ = std::fs::remove_file(&session.wav_path); - // voiceprint.wav + mic_activity.json live and die with audio.wav - // (ADR-0009); never written when retention is off, but remove - // defensively regardless. + // voiceprint.wav lives and dies with audio.wav (ADR-0009); never written + // when retention is off, but remove defensively regardless. let _ = std::fs::remove_file(&voiceprint_path); - let _ = std::fs::remove_file(&mic_activity_path); } else if crate::vault::is_unlocked() { // Seal the retained recording at rest when the vault is unlocked (T8.8). // Runs before the sync enqueue below, so any uploaded copy is ciphertext. @@ -942,23 +915,6 @@ pub async fn stop_recording( } } } - // Phase 3 (FR-SPK): persist the mic "You" timeline so reprocess can - // re-attribute without the live capture. Plaintext timing metadata (no - // audio), local-only (not a sync artifact), lives/dies with audio.wav. - if let Some(mic_activity) = &session.mic_activity { - let file = MicTimelineFile { - schema: 1, - you_spans: mic_activity.you_spans(), - }; - match serde_json::to_string(&file) { - Ok(json) => { - if let Err(e) = std::fs::write(&mic_activity_path, json) { - tracing::warn!("failed to persist mic_activity.json: {e}"); - } - } - Err(e) => tracing::warn!("failed to serialize mic_activity.json: {e}"), - } - } } // Sync-on-finalize (T9.5, FR-SYNC-5): enqueue configured artifacts for @@ -1908,20 +1864,17 @@ pub async fn reprocess_transcript( // diarize the far side masked); otherwise fall back to the whole-signal pass // + voiceprint match (imports, pre-Phase-3 recordings). No models → raw "S1". let mut speaker_names = HashMap::new(); - let you_spans: Option> = - std::fs::read_to_string(meeting_dir(&meeting_id).join("mic_activity.json")) - .ok() - .and_then(|s| serde_json::from_str::(&s).ok()) - .map(|f| f.you_spans); - match (diarizer, you_spans) { - (Some(diarizer), Some(you)) => { - if let Some(names) = - phase3_attribute(diarizer, wav_path.clone(), you, &mut segments).await - { + let is_split = meeting.audio_layout.as_deref() == Some("split"); + match diarizer { + Some(diarizer) if is_split => { + // Split recording: diarize the far side (R channel), "You" from the + // mic (L) channel — recomputed from the file, matching stop's result. + if let Some(names) = attribute_split(diarizer, wav_path.clone(), &mut segments).await { speaker_names = names; } } - (Some(diarizer), None) => { + Some(diarizer) => { + // Summed recording: whole-signal pass + voiceprint "You" match. let mut spans: Option> = None; let diarizer_for_task = diarizer.clone(); let wp = wav_path.clone(); @@ -1950,7 +1903,7 @@ pub async fn reprocess_transcript( } } } - (None, _) => {} // no diarization models: raw "S1" labels + None => {} // no diarization models: raw "S1" labels } let speakers = speaker_infos_from_segments(&segments, &speaker_names); @@ -4960,24 +4913,6 @@ mod tests { assert_eq!(labels, vec!["S2", "S1"]); } - #[test] - fn mask_ranges_zeros_only_the_given_ms_ranges() { - // 1000ms of 16kHz mono = 16_000 samples, all 1.0. - let samples = vec![1.0f32; 16_000]; - // Mask 100..200ms -> samples [1600, 3200). - let out = mask_ranges(samples, &[(100, 200)]); - assert_eq!(out[1599], 1.0); // just before the range - assert_eq!(out[1600], 0.0); // range start zeroed - assert_eq!(out[3199], 0.0); // range end (exclusive) zeroed - assert_eq!(out[3200], 1.0); // just after the range - } - - #[test] - fn mask_ranges_clamps_out_of_bounds_ranges() { - let out = mask_ranges(vec![1.0f32; 100], &[(0, 10_000)]); // way past the end - assert!(out.iter().all(|&s| s == 0.0)); // clamped to len, no panic - } - #[test] fn speaker_infos_applies_display_names_by_label() { let segments = vec![segment("S1"), segment("S2")];