From a14ca871e1397c5e0b9b239e2f67db2f4bfaed85 Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Mon, 6 Jul 2026 18:25:56 -0500 Subject: [PATCH] fix(audio): mix mic into the recording at native rate via MicBridge (FR-CAP-7) --- src-tauri/src/audio/mod.rs | 385 ++++++++++++++++++++++++++++++++++--- 1 file changed, 359 insertions(+), 26 deletions(-) diff --git a/src-tauri/src/audio/mod.rs b/src-tauri/src/audio/mod.rs index 527a823..0908c25 100644 --- a/src-tauri/src/audio/mod.rs +++ b/src-tauri/src/audio/mod.rs @@ -22,9 +22,9 @@ use hound::{SampleFormat, WavSpec, WavWriter}; use std::fs::File; use std::io::BufWriter; use std::path::Path; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::mpsc::SyncSender; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; use wasapi::{ @@ -154,6 +154,63 @@ pub fn list_capture_devices() -> Result, AudioError> { list_devices(&Direction::Capture) } +/// Shared channel the microphone capture fills and the loopback capture drains, +/// so the user's voice is mixed into the recording at the loopback's native rate +/// (FR-CAP-7 in the saved `.wav`, not just the transcript). The loopback thread +/// publishes its `rate` once it opens so the mic knows what to resample to; `buf` +/// is capped to ~0.5s to bound the two independent clocks' drift. +#[cfg(feature = "audio")] +pub struct MicBridge { + rate: AtomicU32, + buf: Mutex>, +} + +#[cfg(feature = "audio")] +impl MicBridge { + pub fn shared() -> Arc { + Arc::new(Self { + rate: AtomicU32::new(0), + buf: Mutex::new(std::collections::VecDeque::new()), + }) + } + + /// Loopback: take `n` mic samples aligned to this callback's frames, padding + /// with silence if the mic hasn't produced enough yet (underrun). + fn pull(&self, n: usize) -> Vec { + let mut out = Vec::with_capacity(n); + if let Ok(mut buf) = self.buf.lock() { + for _ in 0..n { + out.push(buf.pop_front().unwrap_or(0.0)); + } + } else { + out.resize(n, 0.0); + } + out + } + + /// Mic: append resampled samples, dropping the oldest beyond `cap` so a + /// faster mic clock (overrun) can't grow the buffer or the latency unbounded. + fn push(&self, samples: &[f32], cap: usize) { + if let Ok(mut buf) = self.buf.lock() { + buf.extend(samples); + while buf.len() > cap { + buf.pop_front(); + } + } + } +} + +/// Number of audio frames in a raw WASAPI byte buffer of the given format. +#[cfg(feature = "audio")] +fn frame_count(bytes: &[u8], format: &WaveFormat) -> usize { + let bytes_per_frame = (format.get_bitspersample() as usize / 8) * format.get_nchannels() as usize; + if bytes_per_frame == 0 { + 0 + } else { + bytes.len() / bytes_per_frame + } +} + /// Default Windows WASAPI implementation (Phase 1, feature `audio`). #[cfg(feature = "audio")] pub struct WasapiCapture; @@ -164,6 +221,7 @@ impl WasapiCapture { /// `Some` only for the loopback stream (the byte-accurate recording); the mic /// stream produces frames for transcription but no WAV. `direction` selects /// loopback (Render) vs. plain microphone capture (Capture). + #[allow(clippy::too_many_arguments)] fn start_capture( &self, thread_name: &str, @@ -172,6 +230,7 @@ impl WasapiCapture { device_id: Option<&str>, frame_sink: FrameSink, event_sink: EventSink, + bridge: Option>, ) -> Result { let running = Arc::new(AtomicBool::new(true)); let paused = Arc::new(AtomicBool::new(false)); @@ -191,6 +250,7 @@ impl WasapiCapture { &event_sink, &running_th, &paused_th, + bridge.as_ref(), ) }) .map_err(|e| AudioError::Capture(format!("spawn failed: {e}")))?; @@ -201,6 +261,48 @@ impl WasapiCapture { thread, }) } + + /// Loopback capture that mixes the microphone (delivered via `bridge`) into + /// the recording, so the saved `.wav` contains both sides of the meeting at + /// the loopback's native rate/channels (FR-CAP-7). + pub fn start_loopback_recording( + &self, + wav_path: &Path, + device_id: Option<&str>, + frame_sink: FrameSink, + event_sink: EventSink, + bridge: Arc, + ) -> Result { + self.start_capture( + "wa-audio-capture", + Some(wav_path), + Direction::Render, + device_id, + frame_sink, + event_sink, + Some(bridge), + ) + } + + /// Microphone capture that both feeds the live-transcript mixer (`frame_sink`) + /// and pushes its audio into `bridge` for the loopback thread to record. + pub fn start_microphone_recording( + &self, + device_id: Option<&str>, + frame_sink: FrameSink, + event_sink: EventSink, + bridge: Arc, + ) -> Result { + self.start_capture( + "wa-mic-capture", + None, + Direction::Capture, + device_id, + frame_sink, + event_sink, + Some(bridge), + ) + } } #[cfg(feature = "audio")] @@ -219,6 +321,7 @@ impl AudioCapture for WasapiCapture { device_id, frame_sink, event_sink, + None, ) } @@ -235,6 +338,7 @@ impl AudioCapture for WasapiCapture { device_id, frame_sink, event_sink, + None, ) } @@ -378,6 +482,7 @@ const LEVEL_EMIT_INTERVAL: Duration = Duration::from_millis(50); /// WASAPI client and the WAV writer; exits (and finalizes the WAV) once `running` /// is cleared. #[cfg(feature = "audio")] +#[allow(clippy::too_many_arguments)] fn capture_loop( wav_path: Option<&Path>, direction: Direction, @@ -386,15 +491,25 @@ fn capture_loop( event_sink: &EventSink, running: &AtomicBool, paused: &AtomicBool, + bridge: Option<&Arc>, ) -> Result { wasapi::initialize_mta() .ok() .map_err(|e| AudioError::Device(format!("COM init failed: {e}")))?; + let is_loopback = matches!(direction, Direction::Render); // Only the loopback stream emits level updates (FR-CAP-5) and writes a WAV; // the mic stream just contributes frames to the transcript mixer (FR-CAP-7). let emit_level = wav_path.is_some(); let mut session = open_capture_session(&direction, device_id)?; + // Loopback publishes its rate so the mic knows what to resample to before + // pushing into the shared bridge (mic-into-recording, FR-CAP-7). + if is_loopback { + if let Some(b) = bridge { + b.rate + .store(session.format.get_samplespersec(), Ordering::Relaxed); + } + } let spec = wav_spec_for(&session.format)?; let mut writer = match wav_path { Some(path) => Some(WavWriter::create(path, spec).map_err(|e| { @@ -409,6 +524,9 @@ fn capture_loop( .map_err(|e| AudioError::Capture(format!("start_stream failed: {e}")))?; let mut resampler = Resampler::new(session.format.get_samplespersec()); + // Mic-only: resamples the mic to the loopback's rate for the recording bridge + // (created lazily once the loopback has published its rate). + let mut bridge_resampler: Option = None; let mut queue: std::collections::VecDeque = std::collections::VecDeque::new(); let mut frames_written: u64 = 0; let mut last_level_emit = Instant::now() - LEVEL_EMIT_INTERVAL; @@ -440,6 +558,13 @@ fn capture_loop( .start_stream() .map_err(|e| AudioError::Capture(format!("restart_stream failed: {e}")))?; resampler = Resampler::new(new_session.format.get_samplespersec()); + bridge_resampler = None; + if is_loopback { + if let Some(b) = bridge { + b.rate + .store(new_session.format.get_samplespersec(), Ordering::Relaxed); + } + } session = new_session; let _ = event_sink.try_send(CaptureEvent::DeviceChanged { recovered: true, @@ -458,11 +583,34 @@ fn capture_loop( continue; } + // Loopback: mix the mic in per-frame as we write (mic silence when it + // hasn't produced enough), so the recording holds both sides (FR-CAP-7). if let Some(w) = writer.as_mut() { - frames_written += write_wav_bytes(w, &bytes, &session.format)?; + let mic = match bridge { + Some(b) if is_loopback => b.pull(frame_count(&bytes, &session.format)), + _ => Vec::new(), + }; + frames_written += write_wav_bytes(w, &bytes, &session.format, &mic)?; } let mono = decode_mono_f32(&bytes, &session.format)?; + // Mic: feed the shared bridge (resampled to the loopback's rate) so the + // loopback thread can fold it into the recording. + if !is_loopback { + if let Some(b) = bridge { + let rate = b.rate.load(Ordering::Relaxed); + if rate != 0 { + let rs = bridge_resampler.get_or_insert_with(|| { + Resampler::new_to(session.format.get_samplespersec(), rate) + }); + let resampled = rs.process(&mono); + if !resampled.is_empty() { + b.push(&resampled, (rate / 2) as usize); + } + } + } + } + if emit_level && last_level_emit.elapsed() >= LEVEL_EMIT_INTERVAL { let _ = event_sink.try_send(CaptureEvent::Level(audio_level(&mono))); last_level_emit = Instant::now(); @@ -502,42 +650,59 @@ fn wav_spec_for(format: &WaveFormat) -> Result { } /// Write raw WASAPI capture bytes to the WAV writer as 16-bit PCM (FR-CAP-8), -/// quantizing the 32-bit-float mix down on the way so `audio.wav` is ~half the -/// size. Returns the number of frames written. Only the two mix formats WASAPI -/// shared mode actually produces (32-bit float, 16-bit PCM) are supported; -/// anything else is a loud error rather than a silently corrupt recording. +/// quantizing the 32-bit-float mix down so `audio.wav` is ~half the size, and +/// adding one `mic` sample per frame to every channel so the recording carries +/// the user's voice too (FR-CAP-7); `mic` may be shorter than the frame count +/// (missing entries mix as silence) or empty (loopback only). Returns the number +/// of frames written. Only the two mix formats WASAPI shared mode actually +/// produces (32-bit float, 16-bit PCM) are supported; anything else is a loud +/// error rather than a silently corrupt recording. fn write_wav_bytes( writer: &mut WavWriter>, bytes: &[u8], format: &WaveFormat, + mic: &[f32], ) -> Result { let sample_type = format .get_subformat() .map_err(|e| AudioError::Device(format!("unrecognized mix format: {e}")))?; - let channels = format.get_nchannels() as u64; + let channels = format.get_nchannels() as usize; + if channels == 0 { + return Ok(0); + } + let mut frames = 0u64; match (sample_type, format.get_bitspersample()) { (SampleType::Float, 32) => { - for chunk in bytes.chunks_exact(4) { - let v = f32::from_le_bytes(chunk.try_into().unwrap()); - writer - .write_sample(f32_to_i16(v)) - .map_err(|e| AudioError::Capture(format!("wav write: {e}")))?; + for frame in bytes.chunks_exact(4 * channels) { + let add = mic.get(frames as usize).copied().unwrap_or(0.0); + for c in frame.chunks_exact(4) { + let v = f32::from_le_bytes(c.try_into().unwrap()) + add; + writer + .write_sample(f32_to_i16(v)) + .map_err(|e| AudioError::Capture(format!("wav write: {e}")))?; + } + frames += 1; } - Ok(bytes.len() as u64 / 4 / channels.max(1)) } (SampleType::Int, 16) => { - for chunk in bytes.chunks_exact(2) { - let v = i16::from_le_bytes(chunk.try_into().unwrap()); - writer - .write_sample(v) - .map_err(|e| AudioError::Capture(format!("wav write: {e}")))?; + for frame in bytes.chunks_exact(2 * channels) { + let add = mic.get(frames as usize).copied().unwrap_or(0.0); + for c in frame.chunks_exact(2) { + let v = i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32 + add; + writer + .write_sample(f32_to_i16(v)) + .map_err(|e| AudioError::Capture(format!("wav write: {e}")))?; + } + frames += 1; } - Ok(bytes.len() as u64 / 2 / channels.max(1)) } - (st, bits) => Err(AudioError::Device(format!( - "unsupported capture format: {st} {bits}-bit" - ))), + (st, bits) => { + return Err(AudioError::Device(format!( + "unsupported capture format: {st} {bits}-bit" + ))) + } } + Ok(frames) } /// Quantize a `[-1.0, 1.0]` float sample to 16-bit PCM, clamping out-of-range @@ -594,14 +759,22 @@ const TARGET_SAMPLE_RATE: u32 = 16_000; /// resampler if transcription accuracy complaints trace back to aliasing.` struct Resampler { src_rate: u32, + dst_rate: u32, pos: f64, carry: Vec, } impl Resampler { fn new(src_rate: u32) -> Self { + Self::new_to(src_rate, TARGET_SAMPLE_RATE) + } + + /// Resample `src_rate` -> `dst_rate` (used for the mic->loopback-rate bridge + /// as well as the 16kHz transcription path). + fn new_to(src_rate: u32, dst_rate: u32) -> Self { Self { src_rate, + dst_rate, pos: 0.0, carry: Vec::new(), } @@ -611,7 +784,7 @@ impl Resampler { if mono_in.is_empty() { return Vec::new(); } - if self.src_rate == TARGET_SAMPLE_RATE { + if self.src_rate == self.dst_rate { return mono_in.to_vec(); } @@ -622,7 +795,7 @@ impl Resampler { return Vec::new(); } - let ratio = self.src_rate as f64 / TARGET_SAMPLE_RATE as f64; + let ratio = self.src_rate as f64 / self.dst_rate as f64; let mut out = Vec::new(); while (self.pos as usize) + 1 < buf.len() { let i = self.pos as usize; @@ -858,7 +1031,7 @@ mod tests { let mut writer = WavWriter::create(&path, spec).unwrap(); let samples: Vec = (0..2000).map(|i| (i as f32 / 2000.0) - 0.5).collect(); let bytes: Vec = samples.iter().flat_map(|s| s.to_le_bytes()).collect(); - write_wav_bytes(&mut writer, &bytes, &format).unwrap(); + write_wav_bytes(&mut writer, &bytes, &format, &[]).unwrap(); writer.finalize().unwrap(); let reader = hound::WavReader::open(&path).unwrap(); @@ -880,6 +1053,56 @@ mod tests { assert_eq!(f32_to_i16(-2.0), -i16::MAX); // clamped to -32767 } + #[test] + fn write_wav_bytes_mixes_the_mic_into_every_channel() { + // Silent stereo loopback + one mic sample of 0.5 for the single frame + // should land 0.5 (16383) on BOTH channels — the user's voice recorded. + let dir = std::env::temp_dir().join(format!("wa-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("audio.wav"); + let format = WaveFormat::new(32, 32, &SampleType::Float, 48_000, 2, None); + let mut writer = WavWriter::create(&path, wav_spec_for(&format).unwrap()).unwrap(); + // One stereo frame of silence (L=0.0, R=0.0). + let bytes: Vec = [0.0f32, 0.0] + .iter() + .flat_map(|s| s.to_le_bytes()) + .collect(); + let frames = write_wav_bytes(&mut writer, &bytes, &format, &[0.5]).unwrap(); + writer.finalize().unwrap(); + assert_eq!(frames, 1); + + let samples: Vec = hound::WavReader::open(&path) + .unwrap() + .into_samples::() + .map(|s| s.unwrap()) + .collect(); + assert_eq!(samples, vec![16383, 16383]); // mic folded into both channels + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn mic_bridge_pull_pads_with_silence_and_push_caps() { + let bridge = MicBridge::shared(); + // Pull from empty -> all silence. + assert_eq!(bridge.pull(3), vec![0.0, 0.0, 0.0]); + // Push 5 with a cap of 3 -> only the newest 3 survive. + bridge.push(&[1.0, 2.0, 3.0, 4.0, 5.0], 3); + assert_eq!(bridge.pull(3), vec![3.0, 4.0, 5.0]); + // Now empty again -> padded. + assert_eq!(bridge.pull(2), vec![0.0, 0.0]); + } + + #[test] + fn resampler_new_to_downsamples_between_arbitrary_rates() { + // 44.1kHz -> 16kHz still lands on a passthrough when rates match… + let mut same = Resampler::new_to(16_000, 16_000); + assert_eq!(same.process(&[0.1, 0.2, 0.3]), vec![0.1, 0.2, 0.3]); + // …and produces fewer samples when downsampling (48k -> 44.1k). + let mut down = Resampler::new_to(48_000, 44_100); + let out = down.process(&(0..480).map(|i| i as f32).collect::>()); + assert!(out.len() >= 430 && out.len() <= 450, "got {}", out.len()); + } + #[test] fn read_wav_mono_16k_reads_a_normally_finalized_file() { let dir = std::env::temp_dir().join(format!("wa-test-{}", uuid::Uuid::new_v4())); @@ -1036,6 +1259,116 @@ mod tests { assert_eq!(ids.len(), devices.len()); } + #[test] + #[ignore = "captures real loopback while playing a sound; run with --ignored"] + fn loopback_16bit_wav_captures_played_audio() { + // Definitive check that the 32-bit-float -> 16-bit write path (FR-CAP-8) + // records real loopback audio (not silence): start capture, play a known + // WAV through the default device, stop, and assert the 16-bit WAV has + // non-zero samples. + let dir = std::env::temp_dir().join(format!("wa-loop-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("audio.wav"); + let (frame_tx, _frame_rx) = std::sync::mpsc::sync_channel::>(64); + let (event_tx, _event_rx) = std::sync::mpsc::sync_channel::(8); + + let handle = match WasapiCapture.start(&path, None, frame_tx, event_tx) { + Ok(h) => h, + Err(e) => { + eprintln!("[loopback] could not open default render loopback: {e} (skipping)"); + return; + } + }; + // Play an audible system sound through the default render device so + // loopback has something to capture. PlaySync blocks for its duration. + let _ = std::process::Command::new("powershell") + .args([ + "-NoProfile", + "-Command", + "(New-Object Media.SoundPlayer 'C:\\Windows\\Media\\Alarm01.wav').PlaySync()", + ]) + .status(); + std::thread::sleep(Duration::from_millis(500)); + let summary = WasapiCapture.stop(handle).unwrap(); + + let raw = std::fs::read(&path).unwrap(); + let data_at = raw.windows(4).position(|w| w == b"data").unwrap() + 8; + let pcm = &raw[data_at..]; + let max_abs = pcm + .chunks_exact(2) + .map(|c| i16::from_le_bytes([c[0], c[1]]).unsigned_abs()) + .max() + .unwrap_or(0); + eprintln!( + "[loopback] {}Hz {}ch, {} data bytes, peak i16={max_abs}", + summary.sample_rate, + summary.channels, + pcm.len() + ); + let _ = std::fs::remove_dir_all(&dir); + assert!( + max_abs > 0, + "16-bit loopback WAV is silent despite playing audio — real conversion bug" + ); + } + + #[test] + #[ignore = "captures loopback+mic together while playing a sound; run with --ignored"] + fn loopback_recording_with_mic_bridge_captures_played_audio() { + // The exact production path when the mic is on: loopback + mic share a + // MicBridge and the recording must still contain the loopback audio (mic + // silence here just mixes in as 0). Proves the bridge doesn't zero or + // deadlock the recording. + let dir = std::env::temp_dir().join(format!("wa-mixrec-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("audio.wav"); + let bridge = MicBridge::shared(); + let (loop_tx, _lr) = std::sync::mpsc::sync_channel::>(64); + let (mic_tx, _mr) = std::sync::mpsc::sync_channel::>(64); + let (ev_tx, _ev) = std::sync::mpsc::sync_channel::(8); + let (ev_tx2, _ev2) = std::sync::mpsc::sync_channel::(8); + + let loop_h = + match WasapiCapture.start_loopback_recording(&path, None, loop_tx, ev_tx, bridge.clone()) + { + Ok(h) => h, + Err(e) => { + eprintln!("[mixrec] no loopback device: {e} (skipping)"); + return; + } + }; + let mic_h = WasapiCapture + .start_microphone_recording(None, mic_tx, ev_tx2, bridge) + .ok(); + + let _ = std::process::Command::new("powershell") + .args([ + "-NoProfile", + "-Command", + "(New-Object Media.SoundPlayer 'C:\\Windows\\Media\\Alarm01.wav').PlaySync()", + ]) + .status(); + std::thread::sleep(Duration::from_millis(500)); + let summary = WasapiCapture.stop(loop_h).unwrap(); + if let Some(m) = mic_h { + let _ = WasapiCapture.stop(m); + } + + let raw = std::fs::read(&path).unwrap(); + let data_at = raw.windows(4).position(|w| w == b"data").unwrap() + 8; + let peak = raw[data_at..] + .chunks_exact(2) + .map(|c| i16::from_le_bytes([c[0], c[1]]).unsigned_abs()) + .max() + .unwrap_or(0); + eprintln!( + "[mixrec] {}Hz {}ch peak i16={peak}", + summary.sample_rate, summary.channels + ); + let _ = std::fs::remove_dir_all(&dir); + assert!(peak > 0, "combined loopback+mic recording is silent"); + } + #[test] fn find_render_device_resolves_a_real_enumerated_id() { let devices = list_render_devices().unwrap();