iamdoubz/WhispAssist
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

feat(audio): microphone capture + stream mixer (FR-CAP-7)

Browse Source
This commit is contained in:
iamdoubz committed 2026-07-06 16:26:13 -05:00
1 parent b8dc5fc37f
commit 2fb5fd6f30
1 file changed
+266 -38
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/audio/mod.rs
+266 -38
View File
@@ -91,6 +91,17 @@ pub trait AudioCapture: Send + Sync {
frame_sink: FrameSink,
event_sink: EventSink,
) -> Result<CaptureHandle, AudioError>;
/// Capture the user's **microphone** (an input device — normal capture, no
/// loopback) into `frame_sink` for live transcription (FR-CAP-7). No WAV is
/// written here: the byte-accurate recording stays the loopback WAV; the mic
/// is mixed into the transcript stream (see `spawn_mixer`). `device_id` is a
/// `Device::get_id()`; `None` uses the system default capture device.
fn start_microphone(
&self,
device_id: Option<&str>,
frame_sink: FrameSink,
event_sink: EventSink,
) -> Result<CaptureHandle, AudioError>;
fn pause(&self, h: &CaptureHandle) -> Result<(), AudioError>;
fn resume(&self, h: &CaptureHandle) -> Result<(), AudioError>;
fn stop(&self, h: CaptureHandle) -> Result<CaptureSummary, AudioError>;
@@ -104,15 +115,14 @@ pub struct AudioDeviceInfo {
pub name: String,
}
/// Lists active render devices for the "Audio Devices" picker (Settings ->
/// Hardware). Loopback-only, matching the app's only capture path (FR-CAP-1) —
/// there's no microphone picker because there's no microphone capture.
/// Enumerates active devices in one direction (Render = playback for loopback,
/// Capture = microphones) for the Settings device pickers.
#[cfg(feature = "audio")]
pub fn list_render_devices() -> Result<Vec<AudioDeviceInfo>, AudioError> {
fn list_devices(direction: &Direction) -> Result<Vec<AudioDeviceInfo>, AudioError> {
wasapi::initialize_mta()
.ok()
.map_err(|e| AudioError::Device(format!("COM init failed: {e}")))?;
let collection = wasapi::DeviceCollection::new(&Direction::Render)
let collection = wasapi::DeviceCollection::new(direction)
.map_err(|e| AudioError::Device(format!("enumerate devices failed: {e}")))?;
let mut devices = Vec::new();
for device in &collection {
@@ -127,15 +137,36 @@ pub fn list_render_devices() -> Result<Vec<AudioDeviceInfo>, AudioError> {
Ok(devices)
}
/// Lists active render (playback) devices for the loopback "Audio Devices"
/// picker (Settings -> Hardware); WhispAssist records whatever the chosen output
/// plays (FR-CAP-1).
#[cfg(feature = "audio")]
pub fn list_render_devices() -> Result<Vec<AudioDeviceInfo>, AudioError> {
list_devices(&Direction::Render)
}
/// Lists active capture (microphone) devices for the "Microphone" picker
/// (FR-CAP-7) — the user's own voice, mixed into the live transcript.
#[cfg(feature = "audio")]
pub fn list_capture_devices() -> Result<Vec<AudioDeviceInfo>, AudioError> {
list_devices(&Direction::Capture)
}
/// Default Windows WASAPI implementation (Phase 1, feature `audio`).
#[cfg(feature = "audio")]
pub struct WasapiCapture;
#[cfg(feature = "audio")]
impl AudioCapture for WasapiCapture {
fn start(
impl WasapiCapture {
/// Shared spawn path for both loopback and microphone capture. `wav_path` is
/// `Some` only for the loopback stream (the byte-accurate recording); the mic
/// stream produces frames for transcription but no WAV. `direction` selects
/// loopback (Render) vs. plain microphone capture (Capture).
fn start_capture(
&self,
wav_path: &Path,
thread_name: &str,
wav_path: Option<&Path>,
direction: Direction,
device_id: Option<&str>,
frame_sink: FrameSink,
event_sink: EventSink,
@@ -144,14 +175,15 @@ impl AudioCapture for WasapiCapture {
let paused = Arc::new(AtomicBool::new(false));
let running_th = running.clone();
let paused_th = paused.clone();
let wav_path = wav_path.to_path_buf();
let wav_path = wav_path.map(Path::to_path_buf);
let device_id = device_id.map(str::to_string);
let thread = thread::Builder::new()
.name("wa-audio-capture".into())
.name(thread_name.into())
.spawn(move || {
capture_loop(
&wav_path,
wav_path.as_deref(),
direction,
device_id.as_deref(),
&frame_sink,
&event_sink,
@@ -167,6 +199,42 @@ impl AudioCapture for WasapiCapture {
thread,
})
}
}
#[cfg(feature = "audio")]
impl AudioCapture for WasapiCapture {
fn start(
&self,
wav_path: &Path,
device_id: Option<&str>,
frame_sink: FrameSink,
event_sink: EventSink,
) -> Result<CaptureHandle, AudioError> {
self.start_capture(
"wa-audio-capture",
Some(wav_path),
Direction::Render,
device_id,
frame_sink,
event_sink,
)
}
fn start_microphone(
&self,
device_id: Option<&str>,
frame_sink: FrameSink,
event_sink: EventSink,
) -> Result<CaptureHandle, AudioError> {
self.start_capture(
"wa-mic-capture",
None,
Direction::Capture,
device_id,
frame_sink,
event_sink,
)
}
fn pause(&self, h: &CaptureHandle) -> Result<(), AudioError> {
h.paused.store(true, Ordering::SeqCst);
@@ -197,17 +265,17 @@ struct CaptureSession {
format: WaveFormat,
}
/// Resolves the configured render device by id, falling back to the system
/// default if it's no longer present (unplugged/renamed since it was picked) —
/// same "degrade gracefully rather than fail the recording" spirit as the
/// device-recovery reconnect below.
/// Resolves the configured device (in the given direction) by id, falling back
/// to the system default if it's no longer present (unplugged/renamed since it
/// was picked) — same "degrade gracefully rather than fail the recording" spirit
/// as the device-recovery reconnect below.
#[cfg(feature = "audio")]
fn find_render_device(device_id: Option<&str>) -> Result<wasapi::Device, AudioError> {
fn find_device(direction: &Direction, device_id: Option<&str>) -> Result<wasapi::Device, AudioError> {
let Some(id) = device_id else {
return wasapi::get_default_device(&Direction::Render)
.map_err(|e| AudioError::Device(format!("no default render device: {e}")));
return wasapi::get_default_device(direction)
.map_err(|e| AudioError::Device(format!("no default audio device: {e}")));
};
let collection = wasapi::DeviceCollection::new(&Direction::Render)
let collection = wasapi::DeviceCollection::new(direction)
.map_err(|e| AudioError::Device(format!("enumerate devices failed: {e}")))?;
for device in &collection {
let device =
@@ -216,7 +284,7 @@ fn find_render_device(device_id: Option<&str>) -> Result<wasapi::Device, AudioEr
return Ok(device);
}
}
wasapi::get_default_device(&Direction::Render)
wasapi::get_default_device(direction)
.map_err(|e| AudioError::Device(format!("configured device not found and no default: {e}")))
}
@@ -226,8 +294,11 @@ fn find_render_device(device_id: Option<&str>) -> Result<wasapi::Device, AudioEr
/// re-resolves the same selection, so a transient loss reconnects to the same
/// device while a truly-gone device falls back to whatever is now default.
#[cfg(feature = "audio")]
fn open_capture_session(device_id: Option<&str>) -> Result<CaptureSession, AudioError> {
let device = find_render_device(device_id)?;
fn open_capture_session(
direction: &Direction,
device_id: Option<&str>,
) -> Result<CaptureSession, AudioError> {
let device = find_device(direction, device_id)?;
let mut audio_client = device
.get_iaudioclient()
.map_err(|e| AudioError::Device(format!("activate IAudioClient failed: {e}")))?;
@@ -306,7 +377,8 @@ const LEVEL_EMIT_INTERVAL: Duration = Duration::from_millis(50);
/// is cleared.
#[cfg(feature = "audio")]
fn capture_loop(
wav_path: &Path,
wav_path: Option<&Path>,
direction: Direction,
device_id: Option<&str>,
frame_sink: &FrameSink,
event_sink: &EventSink,
@@ -317,11 +389,17 @@ fn capture_loop(
.ok()
.map_err(|e| AudioError::Device(format!("COM init failed: {e}")))?;
let mut session = open_capture_session(device_id)?;
// Only the loopback stream emits level updates (FR-CAP-5) and writes a WAV;
// the mic stream just contributes frames to the transcript mixer (FR-CAP-7).
let emit_level = wav_path.is_some();
let mut session = open_capture_session(&direction, device_id)?;
let spec = wav_spec_for(&session.format)?;
let mut writer = WavWriter::create(wav_path, spec).map_err(|e| {
AudioError::Capture(format!("could not create {}: {e}", wav_path.display()))
})?;
let mut writer = match wav_path {
Some(path) => Some(WavWriter::create(path, spec).map_err(|e| {
AudioError::Capture(format!("could not create {}: {e}", path.display()))
})?),
None => None,
};
session
.audio_client
@@ -347,7 +425,7 @@ fn capture_loop(
message: e.to_string(),
});
session.audio_client.stop_stream().ok();
let new_session = open_capture_session(device_id).map_err(|re| {
let new_session = open_capture_session(&direction, device_id).map_err(|re| {
AudioError::Capture(format!("device lost, reconnect failed: {re}"))
})?;
if !format_compatible(&session.format, &new_session.format) {
@@ -378,10 +456,12 @@ fn capture_loop(
continue;
}
frames_written += write_wav_bytes(&mut writer, &bytes, &session.format)?;
if let Some(w) = writer.as_mut() {
frames_written += write_wav_bytes(w, &bytes, &session.format)?;
}
let mono = decode_mono_f32(&bytes, &session.format)?;
if last_level_emit.elapsed() >= LEVEL_EMIT_INTERVAL {
if emit_level && last_level_emit.elapsed() >= LEVEL_EMIT_INTERVAL {
let _ = event_sink.try_send(CaptureEvent::Level(audio_level(&mono)));
last_level_emit = Instant::now();
}
@@ -393,9 +473,10 @@ fn capture_loop(
}
session.audio_client.stop_stream().ok();
writer
.finalize()
.map_err(|e| AudioError::Capture(format!("wav finalize failed: {e}")))?;
if let Some(w) = writer {
w.finalize()
.map_err(|e| AudioError::Capture(format!("wav finalize failed: {e}")))?;
}
let sample_rate = session.format.get_samplespersec();
Ok(CaptureSummary {
@@ -614,10 +695,145 @@ pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
Ok(resampler.process(&mono))
}
/// Ceiling on how far the still-flowing stream may run ahead of a stalled
/// partner before the mixer forwards it alone — so a muted/dead microphone (or a
/// silent system output) can't hold up the live transcript. 0.5s @ 16kHz.
const MIXER_MAX_LAG: usize = 8_000;
/// Sums two already-16kHz-mono streams (loopback + microphone, FR-CAP-7) sample
/// for sample into the single transcript stream. Pure struct so the alignment
/// logic is unit-testable without threads or real devices.
struct Mixer {
loopback: std::collections::VecDeque<f32>,
mic: std::collections::VecDeque<f32>,
}
impl Mixer {
fn new() -> Self {
Self {
loopback: std::collections::VecDeque::new(),
mic: std::collections::VecDeque::new(),
}
}
/// Emit what's ready: the overlapping head of both buffers summed, plus — if
/// a source has ended (`*_open == false`) or stalled while the other backs up
/// past `MIXER_MAX_LAG` — the leftover of the still-flowing source on its own.
fn drain_ready(&mut self, loop_open: bool, mic_open: bool) -> Vec<f32> {
let n = self.loopback.len().min(self.mic.len());
let mut out = Vec::with_capacity(n);
for _ in 0..n {
let s = self.loopback.pop_front().unwrap() + self.mic.pop_front().unwrap();
// ponytail: hard-clip on sum; only distorts when both sources peak at
// once, and STT tolerates it far better than a wrapped sample.
out.push(s.clamp(-1.0, 1.0));
}
// Microphone gone/stalled -> forward the loopback tail rather than wait.
if !mic_open || (self.mic.is_empty() && self.loopback.len() >= MIXER_MAX_LAG) {
out.extend(self.loopback.drain(..));
}
// Symmetric: loopback gone/stalled -> forward the mic tail.
if !loop_open || (self.loopback.is_empty() && self.mic.len() >= MIXER_MAX_LAG) {
out.extend(self.mic.drain(..));
}
out
}
}
/// Spawns the mixer thread and returns the two frame sinks to hand to the
/// loopback and microphone captures respectively; their summed output flows into
/// `out` (the transcription worker). Only used when the mic is enabled — with it
/// off, loopback frames go straight to `out` as before (no mixer overhead).
pub fn spawn_mixer(out: FrameSink) -> (FrameSink, FrameSink) {
let (loop_tx, loop_rx) = std::sync::mpsc::sync_channel::<Vec<f32>>(8);
let (mic_tx, mic_rx) = std::sync::mpsc::sync_channel::<Vec<f32>>(8);
let _ = thread::Builder::new()
.name("wa-audio-mixer".into())
.spawn(move || mixer_loop(loop_rx, mic_rx, &out));
(loop_tx, mic_tx)
}
fn mixer_loop(
loop_rx: std::sync::mpsc::Receiver<Vec<f32>>,
mic_rx: std::sync::mpsc::Receiver<Vec<f32>>,
out: &FrameSink,
) {
use std::sync::mpsc::{RecvTimeoutError, TryRecvError};
let mut mixer = Mixer::new();
let mut loop_open = true;
let mut mic_open = true;
while loop_open || mic_open || !mixer.loopback.is_empty() || !mixer.mic.is_empty() {
// Loopback is the guaranteed primary source and drives the cadence: block
// on it briefly so we wake on its ~10ms chunks instead of busy-spinning.
if loop_open {
match loop_rx.recv_timeout(Duration::from_millis(20)) {
Ok(chunk) => mixer.loopback.extend(chunk),
Err(RecvTimeoutError::Timeout) => {}
Err(RecvTimeoutError::Disconnected) => loop_open = false,
}
} else {
thread::sleep(Duration::from_millis(20));
}
// Drain whatever the mic has queued without blocking.
loop {
match mic_rx.try_recv() {
Ok(chunk) => mixer.mic.extend(chunk),
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => {
mic_open = false;
break;
}
}
}
let ready = mixer.drain_ready(loop_open, mic_open);
// Blocking send applies backpressure (which then makes the capture threads
// drop frames, as they already do); a send error means the transcription
// consumer is gone, so there's nothing left to mix.
if !ready.is_empty() && out.send(ready).is_err() {
break;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mixer_sums_aligned_streams_and_keeps_the_unmatched_tail() {
let mut m = Mixer::new();
m.loopback.extend([0.1, 0.2, 0.3, 0.4]);
m.mic.extend([0.5, 0.5]);
// Both still open: only the 2-sample overlap is emitted; the loopback tail
// is held back to align with future mic samples (tail < MIXER_MAX_LAG).
let out = m.drain_ready(true, true);
assert_eq!(out, vec![0.6, 0.7]);
assert_eq!(m.loopback.len(), 2);
assert!(m.mic.is_empty());
}
#[test]
fn mixer_clamps_a_summed_overload() {
let mut m = Mixer::new();
m.loopback.extend([0.8, -0.8]);
m.mic.extend([0.8, -0.8]);
let out = m.drain_ready(true, true);
assert_eq!(out, vec![1.0, -1.0]); // 1.6 / -1.6 clamped to the valid range
}
#[test]
fn mixer_flushes_the_survivor_when_the_mic_ends() {
let mut m = Mixer::new();
m.loopback.extend([0.1, 0.2, 0.3]);
m.mic.extend([0.5]);
// Mic closed: sum the 1-sample overlap, then forward the loopback tail
// alone rather than stall the transcript forever.
let out = m.drain_ready(true, false);
assert_eq!(out, vec![0.6, 0.2, 0.3]);
assert!(m.loopback.is_empty());
}
#[test]
fn wav_writer_roundtrip_produces_valid_header_and_duration() {
let dir = std::env::temp_dir().join(format!("wa-test-{}", uuid::Uuid::new_v4()));
@@ -771,13 +987,25 @@ mod tests {
assert_eq!(ids.len(), devices.len());
}
#[test]
fn list_capture_devices_succeeds_and_ids_are_unique() {
// Same contract as the render picker, for the microphone picker
// (FR-CAP-7): enumeration works and ids are stable/unique so
// find_device can resolve a persisted selection.
let devices = list_capture_devices().unwrap();
let mut ids: Vec<&str> = devices.iter().map(|d| d.id.as_str()).collect();
ids.sort_unstable();
ids.dedup();
assert_eq!(ids.len(), devices.len());
}
#[test]
fn find_render_device_resolves_a_real_enumerated_id() {
let devices = list_render_devices().unwrap();
let Some(first) = devices.first() else {
return; // no render device on this machine/CI runner
};
assert!(find_render_device(Some(&first.id)).is_ok());
assert!(find_device(&Direction::Render, Some(&first.id)).is_ok());
}
#[test]
@@ -785,10 +1013,10 @@ mod tests {
// Documents the actual environment rather than assuming one: if this
// machine/CI runner has no default render device at all,
// wasapi::get_default_device itself errors — same as it always has,
// unrelated to device selection — and find_render_device(None) must
// unrelated to device selection — and find_device(None) must
// fail identically rather than pretending to succeed.
let default_ok = wasapi::get_default_device(&Direction::Render).is_ok();
assert_eq!(find_render_device(None).is_ok(), default_ok);
assert_eq!(find_device(&Direction::Render, None).is_ok(), default_ok);
}
#[test]
@@ -796,8 +1024,8 @@ mod tests {
// An id that can never be real should resolve identically to "no
// selection" — same success/failure outcome as whatever this
// environment's default-device lookup actually does.
let none_ok = find_render_device(None).is_ok();
let unknown_ok = find_render_device(Some("not-a-real-device-id")).is_ok();
let none_ok = find_device(&Direction::Render, None).is_ok();
let unknown_ok = find_device(&Direction::Render, Some("not-a-real-device-id")).is_ok();
assert_eq!(none_ok, unknown_ok);
}