From 2fb5fd6f30687f8111d86cfc23323cc8245b8f32 Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Mon, 6 Jul 2026 16:26:13 -0500 Subject: [PATCH] feat(audio): microphone capture + stream mixer (FR-CAP-7) --- src-tauri/src/audio/mod.rs | 304 ++++++++++++++++++++++++++++++++----- 1 file changed, 266 insertions(+), 38 deletions(-) diff --git a/src-tauri/src/audio/mod.rs b/src-tauri/src/audio/mod.rs index a6d4e76..59149ce 100644 --- a/src-tauri/src/audio/mod.rs +++ b/src-tauri/src/audio/mod.rs @@ -91,6 +91,17 @@ pub trait AudioCapture: Send + Sync { frame_sink: FrameSink, event_sink: EventSink, ) -> Result; + /// Capture the user's **microphone** (an input device — normal capture, no + /// loopback) into `frame_sink` for live transcription (FR-CAP-7). No WAV is + /// written here: the byte-accurate recording stays the loopback WAV; the mic + /// is mixed into the transcript stream (see `spawn_mixer`). `device_id` is a + /// `Device::get_id()`; `None` uses the system default capture device. + fn start_microphone( + &self, + device_id: Option<&str>, + frame_sink: FrameSink, + event_sink: EventSink, + ) -> Result; fn pause(&self, h: &CaptureHandle) -> Result<(), AudioError>; fn resume(&self, h: &CaptureHandle) -> Result<(), AudioError>; fn stop(&self, h: CaptureHandle) -> Result; @@ -104,15 +115,14 @@ pub struct AudioDeviceInfo { pub name: String, } -/// Lists active render devices for the "Audio Devices" picker (Settings -> -/// Hardware). Loopback-only, matching the app's only capture path (FR-CAP-1) — -/// there's no microphone picker because there's no microphone capture. +/// Enumerates active devices in one direction (Render = playback for loopback, +/// Capture = microphones) for the Settings device pickers. #[cfg(feature = "audio")] -pub fn list_render_devices() -> Result, AudioError> { +fn list_devices(direction: &Direction) -> Result, AudioError> { wasapi::initialize_mta() .ok() .map_err(|e| AudioError::Device(format!("COM init failed: {e}")))?; - let collection = wasapi::DeviceCollection::new(&Direction::Render) + let collection = wasapi::DeviceCollection::new(direction) .map_err(|e| AudioError::Device(format!("enumerate devices failed: {e}")))?; let mut devices = Vec::new(); for device in &collection { @@ -127,15 +137,36 @@ pub fn list_render_devices() -> Result, AudioError> { Ok(devices) } +/// Lists active render (playback) devices for the loopback "Audio Devices" +/// picker (Settings -> Hardware); WhispAssist records whatever the chosen output +/// plays (FR-CAP-1). +#[cfg(feature = "audio")] +pub fn list_render_devices() -> Result, AudioError> { + list_devices(&Direction::Render) +} + +/// Lists active capture (microphone) devices for the "Microphone" picker +/// (FR-CAP-7) — the user's own voice, mixed into the live transcript. +#[cfg(feature = "audio")] +pub fn list_capture_devices() -> Result, AudioError> { + list_devices(&Direction::Capture) +} + /// Default Windows WASAPI implementation (Phase 1, feature `audio`). #[cfg(feature = "audio")] pub struct WasapiCapture; #[cfg(feature = "audio")] -impl AudioCapture for WasapiCapture { - fn start( +impl WasapiCapture { + /// Shared spawn path for both loopback and microphone capture. `wav_path` is + /// `Some` only for the loopback stream (the byte-accurate recording); the mic + /// stream produces frames for transcription but no WAV. `direction` selects + /// loopback (Render) vs. plain microphone capture (Capture). + fn start_capture( &self, - wav_path: &Path, + thread_name: &str, + wav_path: Option<&Path>, + direction: Direction, device_id: Option<&str>, frame_sink: FrameSink, event_sink: EventSink, @@ -144,14 +175,15 @@ impl AudioCapture for WasapiCapture { let paused = Arc::new(AtomicBool::new(false)); let running_th = running.clone(); let paused_th = paused.clone(); - let wav_path = wav_path.to_path_buf(); + let wav_path = wav_path.map(Path::to_path_buf); let device_id = device_id.map(str::to_string); let thread = thread::Builder::new() - .name("wa-audio-capture".into()) + .name(thread_name.into()) .spawn(move || { capture_loop( - &wav_path, + wav_path.as_deref(), + direction, device_id.as_deref(), &frame_sink, &event_sink, @@ -167,6 +199,42 @@ impl AudioCapture for WasapiCapture { thread, }) } +} + +#[cfg(feature = "audio")] +impl AudioCapture for WasapiCapture { + fn start( + &self, + wav_path: &Path, + device_id: Option<&str>, + frame_sink: FrameSink, + event_sink: EventSink, + ) -> Result { + self.start_capture( + "wa-audio-capture", + Some(wav_path), + Direction::Render, + device_id, + frame_sink, + event_sink, + ) + } + + fn start_microphone( + &self, + device_id: Option<&str>, + frame_sink: FrameSink, + event_sink: EventSink, + ) -> Result { + self.start_capture( + "wa-mic-capture", + None, + Direction::Capture, + device_id, + frame_sink, + event_sink, + ) + } fn pause(&self, h: &CaptureHandle) -> Result<(), AudioError> { h.paused.store(true, Ordering::SeqCst); @@ -197,17 +265,17 @@ struct CaptureSession { format: WaveFormat, } -/// Resolves the configured render device by id, falling back to the system -/// default if it's no longer present (unplugged/renamed since it was picked) — -/// same "degrade gracefully rather than fail the recording" spirit as the -/// device-recovery reconnect below. +/// Resolves the configured device (in the given direction) by id, falling back +/// to the system default if it's no longer present (unplugged/renamed since it +/// was picked) — same "degrade gracefully rather than fail the recording" spirit +/// as the device-recovery reconnect below. #[cfg(feature = "audio")] -fn find_render_device(device_id: Option<&str>) -> Result { +fn find_device(direction: &Direction, device_id: Option<&str>) -> Result { let Some(id) = device_id else { - return wasapi::get_default_device(&Direction::Render) - .map_err(|e| AudioError::Device(format!("no default render device: {e}"))); + return wasapi::get_default_device(direction) + .map_err(|e| AudioError::Device(format!("no default audio device: {e}"))); }; - let collection = wasapi::DeviceCollection::new(&Direction::Render) + let collection = wasapi::DeviceCollection::new(direction) .map_err(|e| AudioError::Device(format!("enumerate devices failed: {e}")))?; for device in &collection { let device = @@ -216,7 +284,7 @@ fn find_render_device(device_id: Option<&str>) -> Result) -> Result) -> Result { - let device = find_render_device(device_id)?; +fn open_capture_session( + direction: &Direction, + device_id: Option<&str>, +) -> Result { + let device = find_device(direction, device_id)?; let mut audio_client = device .get_iaudioclient() .map_err(|e| AudioError::Device(format!("activate IAudioClient failed: {e}")))?; @@ -306,7 +377,8 @@ const LEVEL_EMIT_INTERVAL: Duration = Duration::from_millis(50); /// is cleared. #[cfg(feature = "audio")] fn capture_loop( - wav_path: &Path, + wav_path: Option<&Path>, + direction: Direction, device_id: Option<&str>, frame_sink: &FrameSink, event_sink: &EventSink, @@ -317,11 +389,17 @@ fn capture_loop( .ok() .map_err(|e| AudioError::Device(format!("COM init failed: {e}")))?; - let mut session = open_capture_session(device_id)?; + // Only the loopback stream emits level updates (FR-CAP-5) and writes a WAV; + // the mic stream just contributes frames to the transcript mixer (FR-CAP-7). + let emit_level = wav_path.is_some(); + let mut session = open_capture_session(&direction, device_id)?; let spec = wav_spec_for(&session.format)?; - let mut writer = WavWriter::create(wav_path, spec).map_err(|e| { - AudioError::Capture(format!("could not create {}: {e}", wav_path.display())) - })?; + let mut writer = match wav_path { + Some(path) => Some(WavWriter::create(path, spec).map_err(|e| { + AudioError::Capture(format!("could not create {}: {e}", path.display())) + })?), + None => None, + }; session .audio_client @@ -347,7 +425,7 @@ fn capture_loop( message: e.to_string(), }); session.audio_client.stop_stream().ok(); - let new_session = open_capture_session(device_id).map_err(|re| { + let new_session = open_capture_session(&direction, device_id).map_err(|re| { AudioError::Capture(format!("device lost, reconnect failed: {re}")) })?; if !format_compatible(&session.format, &new_session.format) { @@ -378,10 +456,12 @@ fn capture_loop( continue; } - frames_written += write_wav_bytes(&mut writer, &bytes, &session.format)?; + if let Some(w) = writer.as_mut() { + frames_written += write_wav_bytes(w, &bytes, &session.format)?; + } let mono = decode_mono_f32(&bytes, &session.format)?; - if last_level_emit.elapsed() >= LEVEL_EMIT_INTERVAL { + if emit_level && last_level_emit.elapsed() >= LEVEL_EMIT_INTERVAL { let _ = event_sink.try_send(CaptureEvent::Level(audio_level(&mono))); last_level_emit = Instant::now(); } @@ -393,9 +473,10 @@ fn capture_loop( } session.audio_client.stop_stream().ok(); - writer - .finalize() - .map_err(|e| AudioError::Capture(format!("wav finalize failed: {e}")))?; + if let Some(w) = writer { + w.finalize() + .map_err(|e| AudioError::Capture(format!("wav finalize failed: {e}")))?; + } let sample_rate = session.format.get_samplespersec(); Ok(CaptureSummary { @@ -614,10 +695,145 @@ pub fn read_wav_mono_16k(path: &Path) -> Result, AudioError> { Ok(resampler.process(&mono)) } +/// Ceiling on how far the still-flowing stream may run ahead of a stalled +/// partner before the mixer forwards it alone — so a muted/dead microphone (or a +/// silent system output) can't hold up the live transcript. 0.5s @ 16kHz. +const MIXER_MAX_LAG: usize = 8_000; + +/// Sums two already-16kHz-mono streams (loopback + microphone, FR-CAP-7) sample +/// for sample into the single transcript stream. Pure struct so the alignment +/// logic is unit-testable without threads or real devices. +struct Mixer { + loopback: std::collections::VecDeque, + mic: std::collections::VecDeque, +} + +impl Mixer { + fn new() -> Self { + Self { + loopback: std::collections::VecDeque::new(), + mic: std::collections::VecDeque::new(), + } + } + + /// Emit what's ready: the overlapping head of both buffers summed, plus — if + /// a source has ended (`*_open == false`) or stalled while the other backs up + /// past `MIXER_MAX_LAG` — the leftover of the still-flowing source on its own. + fn drain_ready(&mut self, loop_open: bool, mic_open: bool) -> Vec { + let n = self.loopback.len().min(self.mic.len()); + let mut out = Vec::with_capacity(n); + for _ in 0..n { + let s = self.loopback.pop_front().unwrap() + self.mic.pop_front().unwrap(); + // ponytail: hard-clip on sum; only distorts when both sources peak at + // once, and STT tolerates it far better than a wrapped sample. + out.push(s.clamp(-1.0, 1.0)); + } + // Microphone gone/stalled -> forward the loopback tail rather than wait. + if !mic_open || (self.mic.is_empty() && self.loopback.len() >= MIXER_MAX_LAG) { + out.extend(self.loopback.drain(..)); + } + // Symmetric: loopback gone/stalled -> forward the mic tail. + if !loop_open || (self.loopback.is_empty() && self.mic.len() >= MIXER_MAX_LAG) { + out.extend(self.mic.drain(..)); + } + out + } +} + +/// Spawns the mixer thread and returns the two frame sinks to hand to the +/// loopback and microphone captures respectively; their summed output flows into +/// `out` (the transcription worker). Only used when the mic is enabled — with it +/// off, loopback frames go straight to `out` as before (no mixer overhead). +pub fn spawn_mixer(out: FrameSink) -> (FrameSink, FrameSink) { + let (loop_tx, loop_rx) = std::sync::mpsc::sync_channel::>(8); + let (mic_tx, mic_rx) = std::sync::mpsc::sync_channel::>(8); + let _ = thread::Builder::new() + .name("wa-audio-mixer".into()) + .spawn(move || mixer_loop(loop_rx, mic_rx, &out)); + (loop_tx, mic_tx) +} + +fn mixer_loop( + loop_rx: std::sync::mpsc::Receiver>, + mic_rx: std::sync::mpsc::Receiver>, + out: &FrameSink, +) { + use std::sync::mpsc::{RecvTimeoutError, TryRecvError}; + let mut mixer = Mixer::new(); + let mut loop_open = true; + let mut mic_open = true; + + while loop_open || mic_open || !mixer.loopback.is_empty() || !mixer.mic.is_empty() { + // Loopback is the guaranteed primary source and drives the cadence: block + // on it briefly so we wake on its ~10ms chunks instead of busy-spinning. + if loop_open { + match loop_rx.recv_timeout(Duration::from_millis(20)) { + Ok(chunk) => mixer.loopback.extend(chunk), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => loop_open = false, + } + } else { + thread::sleep(Duration::from_millis(20)); + } + // Drain whatever the mic has queued without blocking. + loop { + match mic_rx.try_recv() { + Ok(chunk) => mixer.mic.extend(chunk), + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + mic_open = false; + break; + } + } + } + let ready = mixer.drain_ready(loop_open, mic_open); + // Blocking send applies backpressure (which then makes the capture threads + // drop frames, as they already do); a send error means the transcription + // consumer is gone, so there's nothing left to mix. + if !ready.is_empty() && out.send(ready).is_err() { + break; + } + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn mixer_sums_aligned_streams_and_keeps_the_unmatched_tail() { + let mut m = Mixer::new(); + m.loopback.extend([0.1, 0.2, 0.3, 0.4]); + m.mic.extend([0.5, 0.5]); + // Both still open: only the 2-sample overlap is emitted; the loopback tail + // is held back to align with future mic samples (tail < MIXER_MAX_LAG). + let out = m.drain_ready(true, true); + assert_eq!(out, vec![0.6, 0.7]); + assert_eq!(m.loopback.len(), 2); + assert!(m.mic.is_empty()); + } + + #[test] + fn mixer_clamps_a_summed_overload() { + let mut m = Mixer::new(); + m.loopback.extend([0.8, -0.8]); + m.mic.extend([0.8, -0.8]); + let out = m.drain_ready(true, true); + assert_eq!(out, vec![1.0, -1.0]); // 1.6 / -1.6 clamped to the valid range + } + + #[test] + fn mixer_flushes_the_survivor_when_the_mic_ends() { + let mut m = Mixer::new(); + m.loopback.extend([0.1, 0.2, 0.3]); + m.mic.extend([0.5]); + // Mic closed: sum the 1-sample overlap, then forward the loopback tail + // alone rather than stall the transcript forever. + let out = m.drain_ready(true, false); + assert_eq!(out, vec![0.6, 0.2, 0.3]); + assert!(m.loopback.is_empty()); + } + #[test] fn wav_writer_roundtrip_produces_valid_header_and_duration() { let dir = std::env::temp_dir().join(format!("wa-test-{}", uuid::Uuid::new_v4())); @@ -771,13 +987,25 @@ mod tests { assert_eq!(ids.len(), devices.len()); } + #[test] + fn list_capture_devices_succeeds_and_ids_are_unique() { + // Same contract as the render picker, for the microphone picker + // (FR-CAP-7): enumeration works and ids are stable/unique so + // find_device can resolve a persisted selection. + let devices = list_capture_devices().unwrap(); + let mut ids: Vec<&str> = devices.iter().map(|d| d.id.as_str()).collect(); + ids.sort_unstable(); + ids.dedup(); + assert_eq!(ids.len(), devices.len()); + } + #[test] fn find_render_device_resolves_a_real_enumerated_id() { let devices = list_render_devices().unwrap(); let Some(first) = devices.first() else { return; // no render device on this machine/CI runner }; - assert!(find_render_device(Some(&first.id)).is_ok()); + assert!(find_device(&Direction::Render, Some(&first.id)).is_ok()); } #[test] @@ -785,10 +1013,10 @@ mod tests { // Documents the actual environment rather than assuming one: if this // machine/CI runner has no default render device at all, // wasapi::get_default_device itself errors — same as it always has, - // unrelated to device selection — and find_render_device(None) must + // unrelated to device selection — and find_device(None) must // fail identically rather than pretending to succeed. let default_ok = wasapi::get_default_device(&Direction::Render).is_ok(); - assert_eq!(find_render_device(None).is_ok(), default_ok); + assert_eq!(find_device(&Direction::Render, None).is_ok(), default_ok); } #[test] @@ -796,8 +1024,8 @@ mod tests { // An id that can never be real should resolve identically to "no // selection" — same success/failure outcome as whatever this // environment's default-device lookup actually does. - let none_ok = find_render_device(None).is_ok(); - let unknown_ok = find_render_device(Some("not-a-real-device-id")).is_ok(); + let none_ok = find_device(&Direction::Render, None).is_ok(); + let unknown_ok = find_device(&Direction::Render, Some("not-a-real-device-id")).is_ok(); assert_eq!(none_ok, unknown_ok); }