iamdoubz/WhispAssist
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

feat(diarization): channel-based split attribution; retire masked-mono path (FR-SPK)

Browse Source 
stop + reprocess now, for a split recording, diarize the right (loopback)
channel into Speaker N and take You from left-channel (mic) VAD — read back from
the file so both agree. Fixes reprocess collapsing to one speaker. Summed
recordings keep the whole-signal + voiceprint fallback. Removes MicTimelineFile,
phase3_attribute, mask_ranges, mic_activity.json (retirement note retained).
This commit is contained in:
iamdoubz committed 2026-07-14 12:16:03 -05:00
1 parent a1d71d1c73
commit 5bd76b80e8
1 file changed
+61 -126
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/commands.rs
+61 -126
View File
@@ -242,44 +242,47 @@ fn diarizer_from_installed_models() -> Option<SherpaDiarizer> {
}
}
/// On-disk `mic_activity.json`: the mic's "You" ranges (Phase 3, FR-SPK),
/// persisted next to a retained `audio.wav` so `reprocess_transcript` can
/// re-attribute "You" without the live capture timeline.
#[derive(serde::Serialize, serde::Deserialize)]
struct MicTimelineFile {
schema: u32,
you_spans: Vec<(u64, u64)>,
}
// RETIRED (2026-07-14): the Phase 3 masked-mono attribution (`MicActivity`
// timeline + `mic_activity.json` + `mask_ranges` + `phase3_attribute`).
// Why it existed: to give per-stream "You" vs "Speaker" attribution when
// `audio.wav` was a summed mono mix — record a live mic-speech timeline, then
// diarize the mix with the mic's ranges masked out so clustering saw only the
// far side. Why retired: it was fragile on reprocess (re-aligning a sidecar
// timeline against a mono mix; collapsed to a single speaker), and the
// dual-channel split layout below makes the mic/far-side separation intrinsic
// to the file instead. Commits: 185489f / fd9311c / 0b5a85f (add), and this
// change (remove). Revisit there if a summed-only capture path is ever needed.
/// Phase 3 (FR-SPK) attribution, shared by `stop_recording` and
/// `reprocess_transcript`: label the mic's `you_spans` "You", diarize the
/// recording with those ranges masked out (so clustering only ever sees the far
/// side), assign the merged spans to `segments`, and return the You + Speaker
/// name map. `None` when the far-side pass fails — the caller then falls back to
/// the whole-signal pass + voiceprint match.
async fn phase3_attribute(
/// Split attribution (FR-SPK, supersedes `phase3_attribute`): diarize the far
/// side (right/loopback channel) into `Speaker N`, take "You" straight from
/// left-channel (mic) voice activity, merge + assign + name. Shared by
/// `stop_recording` and `reprocess_transcript`; both read it back from the file
/// so they agree. `None` if the far-side pass fails (caller falls back).
async fn attribute_split(
diarizer: Arc<dyn Diarizer>,
wav_path: PathBuf,
you_spans: Vec<(u64, u64)>,
segments: &mut [TranscriptSegment],
) -> Option<HashMap<String, String>> {
let diarizer_for_task = diarizer.clone();
let you_for_task = you_spans.clone();
let far = tauri::async_runtime::spawn_blocking(move || {
let samples = crate::audio::read_wav_mono_16k(&wav_path).map_err(|e| e.to_string())?;
diarizer_for_task
.diarize_samples(mask_ranges(samples, &you_for_task))
.map_err(|e| e.to_string())
let result = tauri::async_runtime::spawn_blocking(move || {
// Right channel = loopback/far side; diarize it alone (mic never in it).
let far = crate::audio::read_wav_channel_16k(&wav_path, 1).map_err(|e| e.to_string())?;
let far_spans = diarizer_for_task
.diarize_samples(far)
.map_err(|e| e.to_string())?;
// Left channel = mic; its voice activity is "You".
let mic = crate::audio::read_wav_channel_16k(&wav_path, 0).map_err(|e| e.to_string())?;
Ok::<_, String>((far_spans, crate::audio::vad_spans(&mic)))
})
.await;
let far_spans = match far {
Ok(Ok(spans)) => spans,
let (far_spans, you_spans) = match result {
Ok(Ok(v)) => v,
Ok(Err(e)) => {
tracing::warn!("phase 3 far-side diarization failed: {e}");
tracing::warn!("split attribution failed: {e}");
return None;
}
Err(e) => {
tracing::warn!("phase 3 diarization task failed: {e}");
tracing::warn!("split attribution task failed: {e}");
return None;
}
};
@@ -294,31 +297,11 @@ async fn phase3_attribute(
spans.extend(far_spans);
spans.sort_by_key(|s| s.start_ms);
diarizer.assign(segments, &spans);
// Reuse the voiceprint path's naming so labels are uniform app-wide:
// "You" -> "You", every far speaker in first-appearance order -> "Speaker 2"….
// Uniform naming: "You" -> "You", far speakers -> "Speaker 2", "Speaker 3"….
let labels = crate::diarization::voiceprint::first_appearance_order(&spans);
Some(crate::diarization::voiceprint::build_name_map(&labels, "You"))
}
/// Zero out `ranges` (given as `(start_ms, end_ms)`) in 16kHz mono `samples` —
/// used to remove the mic's "You" ranges from the recording before diarizing the
/// far side, so clustering never sees the mic (Phase 3, FR-SPK).
fn mask_ranges(mut samples: Vec<f32>, ranges: &[(u64, u64)]) -> Vec<f32> {
const SAMPLES_PER_MS: usize = 16; // 16kHz mono
for &(start_ms, end_ms) in ranges {
let start = (start_ms as usize)
.saturating_mul(SAMPLES_PER_MS)
.min(samples.len());
let end = (end_ms as usize)
.saturating_mul(SAMPLES_PER_MS)
.min(samples.len());
if start < end {
samples[start..end].fill(0.0);
}
}
samples
}
/// The distinct speakers seen in `segments` so far, in first-appearance order,
/// with any display names applied (T4.3/T4.4, FR-SPK-2/5). Falls back to the
/// single pre-diarization "S1" placeholder if no segments exist yet.
@@ -416,17 +399,14 @@ pub async fn start_recording(
// microphone that fails to open must not sink the meeting: we log and fall
// back to loopback-only (the mixer forwards loopback alone once its sink
// drops).
let (capture, mic_capture, mic_voice_sample, mic_activity) = if settings.microphone_enabled {
let (capture, mic_capture, mic_voice_sample) = if settings.microphone_enabled {
// The mixer sums both streams for the live transcript; the bridge carries
// the mic into the loopback thread so the recorded WAV holds both sides
// at native quality (FR-CAP-7).
// the mic into the loopback thread. The loopback writes stereo split
// (mic-left/loopback-right) so the two sides stay separate (FR-SPK).
let bridge = crate::audio::MicBridge::shared();
// A few seconds of raw mic audio for the post-stop voiceprint match
// (bug: mic speaker mislabeled "S1"/"S2" instead of "You").
// A few seconds of raw mic audio for the summed-recording voiceprint
// fallback (imports / mic-off diarization).
let voice_sample = crate::audio::VoiceSample::new(MIC_VOICEPRINT_SAMPLES);
// Phase 3 (FR-SPK): timeline of when the mic ("You") is speaking, filled
// by the loopback writer where the mic is folded into audio.wav.
let mic_activity = crate::audio::MicActivity::shared();
let (loop_sink, mic_sink) = crate::audio::spawn_mixer(frame_tx);
let capture = WasapiCapture
.start_loopback_recording(
@@ -435,7 +415,6 @@ pub async fn start_recording(
loop_sink,
event_tx.clone(),
bridge.clone(),
Some(mic_activity.clone()),
)
.map_err(|e| WaError::new("audio", e.to_string()))?;
let mic = WasapiCapture
@@ -449,8 +428,7 @@ pub async fn start_recording(
.map_err(|e| tracing::warn!("microphone capture unavailable: {e}"))
.ok();
let voice_sample = mic.is_some().then_some(voice_sample);
let mic_activity = mic.is_some().then_some(mic_activity);
(capture, mic, voice_sample, mic_activity)
(capture, mic, voice_sample)
} else {
let capture = WasapiCapture
.start(
@@ -460,7 +438,7 @@ pub async fn start_recording(
event_tx,
)
.map_err(|e| WaError::new("audio", e.to_string()))?;
(capture, None, None, None)
(capture, None, None)
};
// Fire-and-forget: exits on its own once `event_tx` drops at capture stop;
@@ -707,7 +685,6 @@ pub async fn start_recording(
diarizer,
speaker_names,
mic_voice_sample,
mic_activity,
// Mic on → the loopback writer records split (mic-left/loopback-right);
// mic off → the summed loopback-only file (FR-SPK).
audio_layout: if settings.microphone_enabled {
@@ -773,22 +750,21 @@ pub async fn stop_recording(
let segment_count = segments.len();
// Speaker attribution over the now-complete recording.
// Phase 3 (FR-SPK): with the mic on, the mic *is* "You" — take "You" spans
// straight from the activity timeline (no clustering) and diarize the far
// side with those ranges masked out, so the mic can never merge into a
// speaker cluster. Falls back to the whole-signal pass (ADR-0005) +
// voiceprint "You" when there's no mic timeline (mic off) or the pass fails.
// FR-SPK: a split recording keeps the mic and far side on separate channels,
// so diarize the far side (right channel) alone → "Speaker N" and take "You"
// from left-channel voice activity. Falls back to the whole-signal pass
// (ADR-0005) + voiceprint "You" for summed recordings (mic off) or if the
// split pass fails.
let mut speaker_names: HashMap<String, String> = HashMap::new();
let mut attributed = false;
if let (Some(diarizer), Some(mic_activity)) =
(session.diarizer.clone(), session.mic_activity.clone())
{
let you = mic_activity.you_spans();
if let Some(names) =
phase3_attribute(diarizer, session.wav_path.clone(), you, &mut segments).await
{
speaker_names = names;
attributed = true;
if session.audio_layout == "split" {
if let Some(diarizer) = session.diarizer.clone() {
if let Some(names) =
attribute_split(diarizer, session.wav_path.clone(), &mut segments).await
{
speaker_names = names;
attributed = true;
}
}
}
@@ -907,14 +883,11 @@ pub async fn stop_recording(
// ADR-0009: delete the working WAV only after the transcript is finalized
// above, and only when retention is off.
let voiceprint_path = meeting_dir(&meeting_id).join("voiceprint.wav");
let mic_activity_path = meeting_dir(&meeting_id).join("mic_activity.json");
if !session.retention {
let _ = std::fs::remove_file(&session.wav_path);
// voiceprint.wav + mic_activity.json live and die with audio.wav
// (ADR-0009); never written when retention is off, but remove
// defensively regardless.
// voiceprint.wav lives and dies with audio.wav (ADR-0009); never written
// when retention is off, but remove defensively regardless.
let _ = std::fs::remove_file(&voiceprint_path);
let _ = std::fs::remove_file(&mic_activity_path);
} else if crate::vault::is_unlocked() {
// Seal the retained recording at rest when the vault is unlocked (T8.8).
// Runs before the sync enqueue below, so any uploaded copy is ciphertext.
@@ -942,23 +915,6 @@ pub async fn stop_recording(
}
}
}
// Phase 3 (FR-SPK): persist the mic "You" timeline so reprocess can
// re-attribute without the live capture. Plaintext timing metadata (no
// audio), local-only (not a sync artifact), lives/dies with audio.wav.
if let Some(mic_activity) = &session.mic_activity {
let file = MicTimelineFile {
schema: 1,
you_spans: mic_activity.you_spans(),
};
match serde_json::to_string(&file) {
Ok(json) => {
if let Err(e) = std::fs::write(&mic_activity_path, json) {
tracing::warn!("failed to persist mic_activity.json: {e}");
}
}
Err(e) => tracing::warn!("failed to serialize mic_activity.json: {e}"),
}
}
}
// Sync-on-finalize (T9.5, FR-SYNC-5): enqueue configured artifacts for
@@ -1908,20 +1864,17 @@ pub async fn reprocess_transcript(
// diarize the far side masked); otherwise fall back to the whole-signal pass
// + voiceprint match (imports, pre-Phase-3 recordings). No models → raw "S1".
let mut speaker_names = HashMap::new();
let you_spans: Option<Vec<(u64, u64)>> =
std::fs::read_to_string(meeting_dir(&meeting_id).join("mic_activity.json"))
.ok()
.and_then(|s| serde_json::from_str::<MicTimelineFile>(&s).ok())
.map(|f| f.you_spans);
match (diarizer, you_spans) {
(Some(diarizer), Some(you)) => {
if let Some(names) =
phase3_attribute(diarizer, wav_path.clone(), you, &mut segments).await
{
let is_split = meeting.audio_layout.as_deref() == Some("split");
match diarizer {
Some(diarizer) if is_split => {
// Split recording: diarize the far side (R channel), "You" from the
// mic (L) channel — recomputed from the file, matching stop's result.
if let Some(names) = attribute_split(diarizer, wav_path.clone(), &mut segments).await {
speaker_names = names;
}
}
(Some(diarizer), None) => {
Some(diarizer) => {
// Summed recording: whole-signal pass + voiceprint "You" match.
let mut spans: Option<Vec<SpeakerSpan>> = None;
let diarizer_for_task = diarizer.clone();
let wp = wav_path.clone();
@@ -1950,7 +1903,7 @@ pub async fn reprocess_transcript(
}
}
}
(None, _) => {} // no diarization models: raw "S1" labels
None => {} // no diarization models: raw "S1" labels
}
let speakers = speaker_infos_from_segments(&segments, &speaker_names);
@@ -4960,24 +4913,6 @@ mod tests {
assert_eq!(labels, vec!["S2", "S1"]);
}
#[test]
fn mask_ranges_zeros_only_the_given_ms_ranges() {
// 1000ms of 16kHz mono = 16_000 samples, all 1.0.
let samples = vec![1.0f32; 16_000];
// Mask 100..200ms -> samples [1600, 3200).
let out = mask_ranges(samples, &[(100, 200)]);
assert_eq!(out[1599], 1.0); // just before the range
assert_eq!(out[1600], 0.0); // range start zeroed
assert_eq!(out[3199], 0.0); // range end (exclusive) zeroed
assert_eq!(out[3200], 1.0); // just after the range
}
#[test]
fn mask_ranges_clamps_out_of_bounds_ranges() {
let out = mask_ranges(vec![1.0f32; 100], &[(0, 10_000)]); // way past the end
assert!(out.iter().all(|&s| s == 0.0)); // clamped to len, no panic
}
#[test]
fn speaker_infos_applies_display_names_by_label() {
let segments = vec![segment("S1"), segment("S2")];