From e287fa5ebe50be9906512c96cf9302a6b2d089da Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Tue, 14 Jul 2026 12:16:01 -0500 Subject: [PATCH] feat(audio): read_wav_channel_16k + vad_spans; retire MicActivity (FR-SPK) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a channel-selective 16k reader (share the decode with read_wav_mono_16k via read_wav_16k) and vad_spans (100ms energy-windowed You detection). Remove the MicActivity live timeline + its capture wiring — the split layout makes the mic/far-side separation intrinsic to the file, so You is recomputed from the left channel instead. --- src-tauri/src/audio/mod.rs | 219 +++++++++++++++---------------------- 1 file changed, 89 insertions(+), 130 deletions(-) diff --git a/src-tauri/src/audio/mod.rs b/src-tauri/src/audio/mod.rs index ce7ba36..0765842 100644 --- a/src-tauri/src/audio/mod.rs +++ b/src-tauri/src/audio/mod.rs @@ -273,86 +273,11 @@ impl VoiceSample { } } -/// Resolution of the mic-activity timeline: one flag per this many ms of audio. -const ACTIVITY_WINDOW_MS: u64 = 100; - -/// Phase 3 (FR-SPK): a per-recording timeline of when the microphone — the user, -/// "You" — was speaking, in `audio.wav` frame time. Captured in the loopback -/// writer (the one place the mic and loopback exist separately in the recording's -/// own timebase), so [`you_spans`](Self::you_spans) lines up with what sherpa -/// reads back for the far-side pass. The far side is then diarized with these -/// ranges masked out, so clustering never sees the mic and can't merge it into a -/// speaker. One flag per [`ACTIVITY_WINDOW_MS`]; ~10 flags/s ≈ negligible memory. -pub struct MicActivity { - /// `audio.wav` (loopback) sample rate; 0 until the loopback stream opens. - rate: AtomicU32, - /// One bool per window; `true` = the mic was speech-level in that window. - windows: Mutex>, -} - -impl MicActivity { - pub fn shared() -> Arc { - Arc::new(Self { - rate: AtomicU32::new(0), - windows: Mutex::new(Vec::new()), - }) - } - - /// The loopback writer publishes its rate so frame positions map to windows. - fn set_rate(&self, rate: u32) { - self.rate.store(rate, Ordering::Relaxed); - } - - /// Flag the `audio.wav` frames `[start_frame, start_frame + mic.len())` as - /// active when the pulled mic chunk is speech-level. Silence is left `false` - /// (the default), so gaps between utterances don't become "You". - fn mark(&self, start_frame: u64, mic: &[f32]) { - let rate = self.rate.load(Ordering::Relaxed) as u64; - if rate == 0 || mic.is_empty() { - return; - } - let rms = (mic.iter().map(|s| s * s).sum::() / mic.len() as f32).sqrt(); - if rms < VOICE_ENERGY_FLOOR { - return; - } - let frames_per_window = (rate * ACTIVITY_WINDOW_MS / 1000).max(1); - let w0 = (start_frame / frames_per_window) as usize; - // Last window actually containing a sample (mic is non-empty here), so a - // chunk ending on a boundary doesn't over-mark the next window. - let w1 = ((start_frame + mic.len() as u64 - 1) / frames_per_window) as usize; - if let Ok(mut win) = self.windows.lock() { - if win.len() <= w1 { - win.resize(w1 + 1, false); - } - win[w0..=w1].fill(true); - } - } - - /// Collapse active windows into merged "You" spans as `(start_ms, end_ms)`. - /// Adjacent active windows coalesce; the caller drops sub-`MIN_SPAN_MS` blips. - pub fn you_spans(&self) -> Vec<(u64, u64)> { - let win = self.windows.lock().map(|w| w.clone()).unwrap_or_default(); - let mut spans = Vec::new(); - let mut start: Option = None; - for (i, &active) in win.iter().enumerate() { - match (active, start) { - (true, None) => start = Some(i), - (false, Some(s)) => { - spans.push((s as u64 * ACTIVITY_WINDOW_MS, i as u64 * ACTIVITY_WINDOW_MS)); - start = None; - } - _ => {} - } - } - if let Some(s) = start { - spans.push(( - s as u64 * ACTIVITY_WINDOW_MS, - win.len() as u64 * ACTIVITY_WINDOW_MS, - )); - } - spans - } -} +// MicActivity (the live mic-speech timeline) was retired 2026-07-14 with the +// masked-mono diarization it fed — the split (mic-left/loopback-right) layout +// makes the mic/far-side separation intrinsic to audio.wav, so "You" is +// recomputed from the left channel via `vad_spans`. See the retirement note in +// commands.rs (attribute_split). /// Number of audio frames in a raw WASAPI byte buffer of the given format. #[cfg(feature = "audio")] @@ -387,7 +312,6 @@ impl WasapiCapture { event_sink: EventSink, bridge: Option>, voice_sample: Option>, - mic_activity: Option>, split: bool, emit_level: bool, ) -> Result { @@ -411,7 +335,6 @@ impl WasapiCapture { &paused_th, bridge.as_ref(), voice_sample.as_ref(), - mic_activity.as_ref(), split, emit_level, ) @@ -435,7 +358,6 @@ impl WasapiCapture { frame_sink: FrameSink, event_sink: EventSink, bridge: Arc, - mic_activity: Option>, ) -> Result { self.start_capture( "wa-audio-capture", @@ -446,9 +368,8 @@ impl WasapiCapture { event_sink, Some(bridge), None, - mic_activity, - true, // split: mic-enabled loopback records mic-left / loopback-right - true, // emit_level + true, // split: mic-enabled loopback records mic-left / loopback-right + true, // emit_level ) } @@ -473,7 +394,6 @@ impl WasapiCapture { event_sink, Some(bridge), voice_sample, - None, false, // split: mic stream has no writer true, // emit_level ) @@ -498,7 +418,6 @@ impl AudioCapture for WasapiCapture { event_sink, None, None, - None, false, // split: loopback-only / mic stream — no split writer true, // emit_level ) @@ -519,7 +438,6 @@ impl AudioCapture for WasapiCapture { event_sink, None, None, - None, false, // split: loopback-only / mic stream — no split writer true, // emit_level ) @@ -694,7 +612,6 @@ fn capture_loop( paused: &AtomicBool, bridge: Option<&Arc>, voice_sample: Option<&Arc>, - mic_activity: Option<&Arc>, // FR-SPK: when true, the loopback WAV is stereo L=mic / R=loopback (the mic // is a separate track, not summed in) so diarization can run on the far side // alone. Only meaningful for the loopback writer. @@ -717,9 +634,6 @@ fn capture_loop( b.rate .store(session.format.get_samplespersec(), Ordering::Relaxed); } - if let Some(ma) = mic_activity { - ma.set_rate(session.format.get_samplespersec()); - } } let mut spec = wav_spec_for(&session.format)?; if split && is_loopback { @@ -781,9 +695,6 @@ fn capture_loop( b.rate .store(new_session.format.get_samplespersec(), Ordering::Relaxed); } - if let Some(ma) = mic_activity { - ma.set_rate(new_session.format.get_samplespersec()); - } } session = new_session; let _ = event_sink.try_send(CaptureEvent::DeviceChanged { @@ -818,15 +729,11 @@ fn capture_loop( let bpf = session.format.get_nchannels() as usize * (session.format.get_bitspersample() as usize / 8); let silence = vec![0u8; gap_frames * bpf]; - let before = frames_written; frames_written += if split { write_wav_split(w, &silence, &session.format, &mic)? } else { write_wav_bytes(w, &silence, &session.format, &mic)? }; - if let Some(ma) = mic_activity { - ma.mark(before, &mic); - } last_write = Instant::now(); } } @@ -849,15 +756,11 @@ fn capture_loop( Some(b) if is_loopback => b.pull(frame_count(&bytes, &session.format)), _ => Vec::new(), }; - let before = frames_written; frames_written += if split { write_wav_split(w, &bytes, &session.format, &mic)? } else { write_wav_bytes(w, &bytes, &session.format, &mic)? }; - if let Some(ma) = mic_activity { - ma.mark(before, &mic); - } } let mono = decode_mono_f32(&bytes, &session.format)?; @@ -1148,6 +1051,22 @@ impl Resampler { /// through hound's size-aware iterator; hound is only used to parse the /// format/`fmt ` chunk. pub fn read_wav_mono_16k(path: &Path) -> Result, AudioError> { + read_wav_16k(path, None) +} + +/// Like [`read_wav_mono_16k`] but extracts a single channel instead of +/// downmixing — used by the Phase 3.5 split path to diarize the right +/// (loopback) channel and VAD the left (mic) channel (FR-SPK). `channel` is +/// clamped to the file's last channel, so it's safe on a mono/summed file. +pub fn read_wav_channel_16k(path: &Path, channel: usize) -> Result, AudioError> { + read_wav_16k(path, Some(channel)) +} + +/// Shared reader: decrypt (T8.8), reduce each frame to one f32 — averaging all +/// channels (`channel = None`) or picking one (`channel = Some`) — resample to +/// 16kHz. Reads the `data` payload as raw bytes (an unfinalized WAV's size +/// fields read back as 0), so it works on a mid-write / crash-left file (FR-REL-1). +fn read_wav_16k(path: &Path, channel: Option) -> Result, AudioError> { let raw = std::fs::read(path).map_err(|e| AudioError::Capture(format!("read wav: {e}")))?; // Decrypt if the recording was sealed at rest (T8.8); plaintext passes through. let file = @@ -1157,6 +1076,8 @@ pub fn read_wav_mono_16k(path: &Path) -> Result, AudioError> { .map_err(|e| AudioError::Capture(format!("open wav: {e}")))? .spec(); let channels = spec.channels.max(1) as usize; + // Which channel index to pick (clamped), or None to downmix. + let pick = channel.map(|c| c.min(channels - 1)); let data_marker = file .windows(4) @@ -1168,22 +1089,14 @@ pub fn read_wav_mono_16k(path: &Path) -> Result, AudioError> { let mono: Vec = match (spec.sample_format, spec.bits_per_sample) { (SampleFormat::Float, 32) => bytes .chunks_exact(4 * channels) - .map(|frame| { - frame - .chunks_exact(4) - .map(|c| f32::from_le_bytes(c.try_into().unwrap())) - .sum::() - / channels as f32 - }) + .map(|frame| reduce_frame(frame, 4, pick, channels, |c| f32::from_le_bytes(c.try_into().unwrap()))) .collect(), (SampleFormat::Int, 16) => bytes .chunks_exact(2 * channels) .map(|frame| { - frame - .chunks_exact(2) - .map(|c| i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32) - .sum::() - / channels as f32 + reduce_frame(frame, 2, pick, channels, |c| { + i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32 + }) }) .collect(), (fmt, bits) => { @@ -1197,6 +1110,56 @@ pub fn read_wav_mono_16k(path: &Path) -> Result, AudioError> { Ok(resampler.process(&mono)) } +/// Voice-activity "You" spans (FR-SPK): 100ms windows of 16kHz mono audio above +/// [`VOICE_ENERGY_FLOOR`], merged into `(start_ms, end_ms)` ranges. Run on the +/// left (mic) channel of a split recording to attribute segments to "You" — +/// replaces the old live `MicActivity` timeline (recomputed from the file, so +/// stop and reprocess agree). +pub fn vad_spans(samples: &[f32]) -> Vec<(u64, u64)> { + const WINDOW_MS: u64 = 100; + let win = (TARGET_SAMPLE_RATE as u64 * WINDOW_MS / 1000) as usize; // 1600 @16kHz + if win == 0 { + return Vec::new(); + } + let mut spans = Vec::new(); + let mut start: Option = None; + let mut w = 0u64; + let mut i = 0usize; + while i < samples.len() { + let chunk = &samples[i..(i + win).min(samples.len())]; + let rms = (chunk.iter().map(|s| s * s).sum::() / chunk.len().max(1) as f32).sqrt(); + match (rms >= VOICE_ENERGY_FLOOR, start) { + (true, None) => start = Some(w * WINDOW_MS), + (false, Some(s)) => { + spans.push((s, w * WINDOW_MS)); + start = None; + } + _ => {} + } + i += win; + w += 1; + } + if let Some(s) = start { + spans.push((s, w * WINDOW_MS)); + } + spans +} + +/// Reduce one interleaved frame to a single f32: pick channel `pick` or average +/// all `channels`. `width` is bytes-per-sample; `decode` turns those bytes → f32. +fn reduce_frame( + frame: &[u8], + width: usize, + pick: Option, + channels: usize, + decode: impl Fn(&[u8]) -> f32, +) -> f32 { + match pick { + Some(c) => decode(&frame[c * width..c * width + width]), + None => frame.chunks_exact(width).map(decode).sum::() / channels as f32, + } +} + /// Write mono 16kHz `f32` samples as a Float32 WAV — the format /// [`read_wav_mono_16k`] reads back. Used to persist the mic voiceprint sample /// next to a retained recording so it can be re-matched on reprocess (FR-SPK). @@ -1325,22 +1288,19 @@ mod tests { use super::*; #[test] - fn mic_activity_ignores_silence() { - let ma = MicActivity::shared(); - ma.set_rate(16_000); - ma.mark(0, &[0.0; 3_200]); // below the energy floor - assert!(ma.you_spans().is_empty()); + fn vad_spans_ignores_silence() { + // 200ms of 16kHz silence → no "You" spans. + assert!(vad_spans(&[0.0; 3_200]).is_empty()); } #[test] - fn mic_activity_collapses_speech_into_spans_with_gaps() { - let ma = MicActivity::shared(); - ma.set_rate(16_000); // 1600 frames per 100ms window - let speech = vec![0.2f32; 3_200]; // 200ms of speech - ma.mark(0, &speech); // frames [0,3200) -> windows 0,1 (ms 0-200) - // gap: windows 2,3 (ms 200-400) left silent - ma.mark(6_400, &speech); // frames [6400,9600) -> windows 4,5 (ms 400-600) - assert_eq!(ma.you_spans(), vec![(0, 200), (400, 600)]); + fn vad_spans_collapses_speech_into_spans_with_gaps() { + // 16kHz: 1600 samples per 100ms window. Speech in windows 0,1 (0-200ms) + // and 4,5 (400-600ms), silence between. + let mut samples = vec![0.0f32; 9_600]; // 600ms + samples[0..3_200].fill(0.2); // windows 0,1 + samples[6_400..9_600].fill(0.2); // windows 4,5 + assert_eq!(vad_spans(&samples), vec![(0, 200), (400, 600)]); } #[test] @@ -1741,7 +1701,6 @@ mod tests { loop_tx, ev_tx, bridge.clone(), - None, ) { Ok(h) => h, Err(e) => {