iamdoubz/WhispAssist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

fix(audio): mix mic into the recording at native rate via MicBridge (FR-CAP-7)

Browse Source
This commit is contained in:
iamdoubz
2026-07-06 18:25:56 -05:00
parent 2fda4041ff
commit a14ca871e1
1 changed files with 359 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/audio/mod.rs
+359 -26
View File
@@ -22,9 +22,9 @@ use hound::{SampleFormat, WavSpec, WavWriter};
use std::fs::File;
use std::io::BufWriter;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::mpsc::SyncSender;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use wasapi::{
@@ -154,6 +154,63 @@ pub fn list_capture_devices() -> Result<Vec<AudioDeviceInfo>, AudioError> {
list_devices(&Direction::Capture)
}
/// Shared channel the microphone capture fills and the loopback capture drains,
/// so the user's voice is mixed into the recording at the loopback's native rate
/// (FR-CAP-7 in the saved `.wav`, not just the transcript). The loopback thread
/// publishes its `rate` once it opens so the mic knows what to resample to; `buf`
/// is capped to ~0.5s to bound the two independent clocks' drift.
#[cfg(feature = "audio")]
pub struct MicBridge {
rate: AtomicU32,
buf: Mutex<std::collections::VecDeque<f32>>,
}
#[cfg(feature = "audio")]
impl MicBridge {
pub fn shared() -> Arc<Self> {
Arc::new(Self {
rate: AtomicU32::new(0),
buf: Mutex::new(std::collections::VecDeque::new()),
})
}
/// Loopback: take `n` mic samples aligned to this callback's frames, padding
/// with silence if the mic hasn't produced enough yet (underrun).
fn pull(&self, n: usize) -> Vec<f32> {
let mut out = Vec::with_capacity(n);
if let Ok(mut buf) = self.buf.lock() {
for _ in 0..n {
out.push(buf.pop_front().unwrap_or(0.0));
}
} else {
out.resize(n, 0.0);
}
out
}
/// Mic: append resampled samples, dropping the oldest beyond `cap` so a
/// faster mic clock (overrun) can't grow the buffer or the latency unbounded.
fn push(&self, samples: &[f32], cap: usize) {
if let Ok(mut buf) = self.buf.lock() {
buf.extend(samples);
while buf.len() > cap {
buf.pop_front();
}
}
}
}
/// Number of audio frames in a raw WASAPI byte buffer of the given format.
#[cfg(feature = "audio")]
fn frame_count(bytes: &[u8], format: &WaveFormat) -> usize {
let bytes_per_frame = (format.get_bitspersample() as usize / 8) * format.get_nchannels() as usize;
if bytes_per_frame == 0 {
0
} else {
bytes.len() / bytes_per_frame
}
}
/// Default Windows WASAPI implementation (Phase 1, feature `audio`).
#[cfg(feature = "audio")]
pub struct WasapiCapture;
@@ -164,6 +221,7 @@ impl WasapiCapture {
/// `Some` only for the loopback stream (the byte-accurate recording); the mic
/// stream produces frames for transcription but no WAV. `direction` selects
/// loopback (Render) vs. plain microphone capture (Capture).
#[allow(clippy::too_many_arguments)]
fn start_capture(
&self,
thread_name: &str,
@@ -172,6 +230,7 @@ impl WasapiCapture {
device_id: Option<&str>,
frame_sink: FrameSink,
event_sink: EventSink,
bridge: Option<Arc<MicBridge>>,
) -> Result<CaptureHandle, AudioError> {
let running = Arc::new(AtomicBool::new(true));
let paused = Arc::new(AtomicBool::new(false));
@@ -191,6 +250,7 @@ impl WasapiCapture {
&event_sink,
&running_th,
&paused_th,
bridge.as_ref(),
)
})
.map_err(|e| AudioError::Capture(format!("spawn failed: {e}")))?;
@@ -201,6 +261,48 @@ impl WasapiCapture {
thread,
})
}
/// Loopback capture that mixes the microphone (delivered via `bridge`) into
/// the recording, so the saved `.wav` contains both sides of the meeting at
/// the loopback's native rate/channels (FR-CAP-7).
pub fn start_loopback_recording(
&self,
wav_path: &Path,
device_id: Option<&str>,
frame_sink: FrameSink,
event_sink: EventSink,
bridge: Arc<MicBridge>,
) -> Result<CaptureHandle, AudioError> {
self.start_capture(
"wa-audio-capture",
Some(wav_path),
Direction::Render,
device_id,
frame_sink,
event_sink,
Some(bridge),
)
}
/// Microphone capture that both feeds the live-transcript mixer (`frame_sink`)
/// and pushes its audio into `bridge` for the loopback thread to record.
pub fn start_microphone_recording(
&self,
device_id: Option<&str>,
frame_sink: FrameSink,
event_sink: EventSink,
bridge: Arc<MicBridge>,
) -> Result<CaptureHandle, AudioError> {
self.start_capture(
"wa-mic-capture",
None,
Direction::Capture,
device_id,
frame_sink,
event_sink,
Some(bridge),
)
}
}
#[cfg(feature = "audio")]
@@ -219,6 +321,7 @@ impl AudioCapture for WasapiCapture {
device_id,
frame_sink,
event_sink,
None,
)
}
@@ -235,6 +338,7 @@ impl AudioCapture for WasapiCapture {
device_id,
frame_sink,
event_sink,
None,
)
}
@@ -378,6 +482,7 @@ const LEVEL_EMIT_INTERVAL: Duration = Duration::from_millis(50);
/// WASAPI client and the WAV writer; exits (and finalizes the WAV) once `running`
/// is cleared.
#[cfg(feature = "audio")]
#[allow(clippy::too_many_arguments)]
fn capture_loop(
wav_path: Option<&Path>,
direction: Direction,
@@ -386,15 +491,25 @@ fn capture_loop(
event_sink: &EventSink,
running: &AtomicBool,
paused: &AtomicBool,
bridge: Option<&Arc<MicBridge>>,
) -> Result<CaptureSummary, AudioError> {
wasapi::initialize_mta()
.ok()
.map_err(|e| AudioError::Device(format!("COM init failed: {e}")))?;
let is_loopback = matches!(direction, Direction::Render);
// Only the loopback stream emits level updates (FR-CAP-5) and writes a WAV;
// the mic stream just contributes frames to the transcript mixer (FR-CAP-7).
let emit_level = wav_path.is_some();
let mut session = open_capture_session(&direction, device_id)?;
// Loopback publishes its rate so the mic knows what to resample to before
// pushing into the shared bridge (mic-into-recording, FR-CAP-7).
if is_loopback {
if let Some(b) = bridge {
b.rate
.store(session.format.get_samplespersec(), Ordering::Relaxed);
}
}
let spec = wav_spec_for(&session.format)?;
let mut writer = match wav_path {
Some(path) => Some(WavWriter::create(path, spec).map_err(|e| {
@@ -409,6 +524,9 @@ fn capture_loop(
.map_err(|e| AudioError::Capture(format!("start_stream failed: {e}")))?;
let mut resampler = Resampler::new(session.format.get_samplespersec());
// Mic-only: resamples the mic to the loopback's rate for the recording bridge
// (created lazily once the loopback has published its rate).
let mut bridge_resampler: Option<Resampler> = None;
let mut queue: std::collections::VecDeque<u8> = std::collections::VecDeque::new();
let mut frames_written: u64 = 0;
let mut last_level_emit = Instant::now() - LEVEL_EMIT_INTERVAL;
@@ -440,6 +558,13 @@ fn capture_loop(
.start_stream()
.map_err(|e| AudioError::Capture(format!("restart_stream failed: {e}")))?;
resampler = Resampler::new(new_session.format.get_samplespersec());
bridge_resampler = None;
if is_loopback {
if let Some(b) = bridge {
b.rate
.store(new_session.format.get_samplespersec(), Ordering::Relaxed);
}
}
session = new_session;
let _ = event_sink.try_send(CaptureEvent::DeviceChanged {
recovered: true,
@@ -458,11 +583,34 @@ fn capture_loop(
continue;
}
// Loopback: mix the mic in per-frame as we write (mic silence when it
// hasn't produced enough), so the recording holds both sides (FR-CAP-7).
if let Some(w) = writer.as_mut() {
frames_written += write_wav_bytes(w, &bytes, &session.format)?;
let mic = match bridge {
Some(b) if is_loopback => b.pull(frame_count(&bytes, &session.format)),
_ => Vec::new(),
};
frames_written += write_wav_bytes(w, &bytes, &session.format, &mic)?;
}
let mono = decode_mono_f32(&bytes, &session.format)?;
// Mic: feed the shared bridge (resampled to the loopback's rate) so the
// loopback thread can fold it into the recording.
if !is_loopback {
if let Some(b) = bridge {
let rate = b.rate.load(Ordering::Relaxed);
if rate != 0 {
let rs = bridge_resampler.get_or_insert_with(|| {
Resampler::new_to(session.format.get_samplespersec(), rate)
});
let resampled = rs.process(&mono);
if !resampled.is_empty() {
b.push(&resampled, (rate / 2) as usize);
}
}
}
}
if emit_level && last_level_emit.elapsed() >= LEVEL_EMIT_INTERVAL {
let _ = event_sink.try_send(CaptureEvent::Level(audio_level(&mono)));
last_level_emit = Instant::now();
@@ -502,42 +650,59 @@ fn wav_spec_for(format: &WaveFormat) -> Result<WavSpec, AudioError> {
}
/// Write raw WASAPI capture bytes to the WAV writer as 16-bit PCM (FR-CAP-8),
/// quantizing the 32-bit-float mix down on the way so `audio.wav` is ~half the
/// size. Returns the number of frames written. Only the two mix formats WASAPI
/// shared mode actually produces (32-bit float, 16-bit PCM) are supported;
/// anything else is a loud error rather than a silently corrupt recording.
/// quantizing the 32-bit-float mix down so `audio.wav` is ~half the size, and
/// adding one `mic` sample per frame to every channel so the recording carries
/// the user's voice too (FR-CAP-7); `mic` may be shorter than the frame count
/// (missing entries mix as silence) or empty (loopback only). Returns the number
/// of frames written. Only the two mix formats WASAPI shared mode actually
/// produces (32-bit float, 16-bit PCM) are supported; anything else is a loud
/// error rather than a silently corrupt recording.
fn write_wav_bytes(
writer: &mut WavWriter<BufWriter<File>>,
bytes: &[u8],
format: &WaveFormat,
mic: &[f32],
) -> Result<u64, AudioError> {
let sample_type = format
.get_subformat()
.map_err(|e| AudioError::Device(format!("unrecognized mix format: {e}")))?;
let channels = format.get_nchannels() as u64;
let channels = format.get_nchannels() as usize;
if channels == 0 {
return Ok(0);
}
let mut frames = 0u64;
match (sample_type, format.get_bitspersample()) {
(SampleType::Float, 32) => {
for chunk in bytes.chunks_exact(4) {
let v = f32::from_le_bytes(chunk.try_into().unwrap());
writer
.write_sample(f32_to_i16(v))
.map_err(|e| AudioError::Capture(format!("wav write: {e}")))?;
for frame in bytes.chunks_exact(4 * channels) {
let add = mic.get(frames as usize).copied().unwrap_or(0.0);
for c in frame.chunks_exact(4) {
let v = f32::from_le_bytes(c.try_into().unwrap()) + add;
writer
.write_sample(f32_to_i16(v))
.map_err(|e| AudioError::Capture(format!("wav write: {e}")))?;
}
frames += 1;
}
Ok(bytes.len() as u64 / 4 / channels.max(1))
}
(SampleType::Int, 16) => {
for chunk in bytes.chunks_exact(2) {
let v = i16::from_le_bytes(chunk.try_into().unwrap());
writer
.write_sample(v)
.map_err(|e| AudioError::Capture(format!("wav write: {e}")))?;
for frame in bytes.chunks_exact(2 * channels) {
let add = mic.get(frames as usize).copied().unwrap_or(0.0);
for c in frame.chunks_exact(2) {
let v = i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32 + add;
writer
.write_sample(f32_to_i16(v))
.map_err(|e| AudioError::Capture(format!("wav write: {e}")))?;
}
frames += 1;
}
Ok(bytes.len() as u64 / 2 / channels.max(1))
}
(st, bits) => Err(AudioError::Device(format!(
"unsupported capture format: {st} {bits}-bit"
))),
(st, bits) => {
return Err(AudioError::Device(format!(
"unsupported capture format: {st} {bits}-bit"
)))
}
}
Ok(frames)
}
/// Quantize a `[-1.0, 1.0]` float sample to 16-bit PCM, clamping out-of-range
@@ -594,14 +759,22 @@ const TARGET_SAMPLE_RATE: u32 = 16_000;
/// resampler if transcription accuracy complaints trace back to aliasing.`
struct Resampler {
src_rate: u32,
dst_rate: u32,
pos: f64,
carry: Vec<f32>,
}
impl Resampler {
fn new(src_rate: u32) -> Self {
Self::new_to(src_rate, TARGET_SAMPLE_RATE)
}
/// Resample `src_rate` -> `dst_rate` (used for the mic->loopback-rate bridge
/// as well as the 16kHz transcription path).
fn new_to(src_rate: u32, dst_rate: u32) -> Self {
Self {
src_rate,
dst_rate,
pos: 0.0,
carry: Vec::new(),
}
@@ -611,7 +784,7 @@ impl Resampler {
if mono_in.is_empty() {
return Vec::new();
}
if self.src_rate == TARGET_SAMPLE_RATE {
if self.src_rate == self.dst_rate {
return mono_in.to_vec();
}
@@ -622,7 +795,7 @@ impl Resampler {
return Vec::new();
}
let ratio = self.src_rate as f64 / TARGET_SAMPLE_RATE as f64;
let ratio = self.src_rate as f64 / self.dst_rate as f64;
let mut out = Vec::new();
while (self.pos as usize) + 1 < buf.len() {
let i = self.pos as usize;
@@ -858,7 +1031,7 @@ mod tests {
let mut writer = WavWriter::create(&path, spec).unwrap();
let samples: Vec<f32> = (0..2000).map(|i| (i as f32 / 2000.0) - 0.5).collect();
let bytes: Vec<u8> = samples.iter().flat_map(|s| s.to_le_bytes()).collect();
write_wav_bytes(&mut writer, &bytes, &format).unwrap();
write_wav_bytes(&mut writer, &bytes, &format, &[]).unwrap();
writer.finalize().unwrap();
let reader = hound::WavReader::open(&path).unwrap();
@@ -880,6 +1053,56 @@ mod tests {
assert_eq!(f32_to_i16(-2.0), -i16::MAX); // clamped to -32767
}
#[test]
fn write_wav_bytes_mixes_the_mic_into_every_channel() {
// Silent stereo loopback + one mic sample of 0.5 for the single frame
// should land 0.5 (16383) on BOTH channels — the user's voice recorded.
let dir = std::env::temp_dir().join(format!("wa-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("audio.wav");
let format = WaveFormat::new(32, 32, &SampleType::Float, 48_000, 2, None);
let mut writer = WavWriter::create(&path, wav_spec_for(&format).unwrap()).unwrap();
// One stereo frame of silence (L=0.0, R=0.0).
let bytes: Vec<u8> = [0.0f32, 0.0]
.iter()
.flat_map(|s| s.to_le_bytes())
.collect();
let frames = write_wav_bytes(&mut writer, &bytes, &format, &[0.5]).unwrap();
writer.finalize().unwrap();
assert_eq!(frames, 1);
let samples: Vec<i16> = hound::WavReader::open(&path)
.unwrap()
.into_samples::<i16>()
.map(|s| s.unwrap())
.collect();
assert_eq!(samples, vec![16383, 16383]); // mic folded into both channels
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn mic_bridge_pull_pads_with_silence_and_push_caps() {
let bridge = MicBridge::shared();
// Pull from empty -> all silence.
assert_eq!(bridge.pull(3), vec![0.0, 0.0, 0.0]);
// Push 5 with a cap of 3 -> only the newest 3 survive.
bridge.push(&[1.0, 2.0, 3.0, 4.0, 5.0], 3);
assert_eq!(bridge.pull(3), vec![3.0, 4.0, 5.0]);
// Now empty again -> padded.
assert_eq!(bridge.pull(2), vec![0.0, 0.0]);
}
#[test]
fn resampler_new_to_downsamples_between_arbitrary_rates() {
// 44.1kHz -> 16kHz still lands on a passthrough when rates match…
let mut same = Resampler::new_to(16_000, 16_000);
assert_eq!(same.process(&[0.1, 0.2, 0.3]), vec![0.1, 0.2, 0.3]);
// …and produces fewer samples when downsampling (48k -> 44.1k).
let mut down = Resampler::new_to(48_000, 44_100);
let out = down.process(&(0..480).map(|i| i as f32).collect::<Vec<_>>());
assert!(out.len() >= 430 && out.len() <= 450, "got {}", out.len());
}
#[test]
fn read_wav_mono_16k_reads_a_normally_finalized_file() {
let dir = std::env::temp_dir().join(format!("wa-test-{}", uuid::Uuid::new_v4()));
@@ -1036,6 +1259,116 @@ mod tests {
assert_eq!(ids.len(), devices.len());
}
#[test]
#[ignore = "captures real loopback while playing a sound; run with --ignored"]
fn loopback_16bit_wav_captures_played_audio() {
// Definitive check that the 32-bit-float -> 16-bit write path (FR-CAP-8)
// records real loopback audio (not silence): start capture, play a known
// WAV through the default device, stop, and assert the 16-bit WAV has
// non-zero samples.
let dir = std::env::temp_dir().join(format!("wa-loop-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("audio.wav");
let (frame_tx, _frame_rx) = std::sync::mpsc::sync_channel::<Vec<f32>>(64);
let (event_tx, _event_rx) = std::sync::mpsc::sync_channel::<CaptureEvent>(8);
let handle = match WasapiCapture.start(&path, None, frame_tx, event_tx) {
Ok(h) => h,
Err(e) => {
eprintln!("[loopback] could not open default render loopback: {e} (skipping)");
return;
}
};
// Play an audible system sound through the default render device so
// loopback has something to capture. PlaySync blocks for its duration.
let _ = std::process::Command::new("powershell")
.args([
"-NoProfile",
"-Command",
"(New-Object Media.SoundPlayer 'C:\\Windows\\Media\\Alarm01.wav').PlaySync()",
])
.status();
std::thread::sleep(Duration::from_millis(500));
let summary = WasapiCapture.stop(handle).unwrap();
let raw = std::fs::read(&path).unwrap();
let data_at = raw.windows(4).position(|w| w == b"data").unwrap() + 8;
let pcm = &raw[data_at..];
let max_abs = pcm
.chunks_exact(2)
.map(|c| i16::from_le_bytes([c[0], c[1]]).unsigned_abs())
.max()
.unwrap_or(0);
eprintln!(
"[loopback] {}Hz {}ch, {} data bytes, peak i16={max_abs}",
summary.sample_rate,
summary.channels,
pcm.len()
);
let _ = std::fs::remove_dir_all(&dir);
assert!(
max_abs > 0,
"16-bit loopback WAV is silent despite playing audio — real conversion bug"
);
}
#[test]
#[ignore = "captures loopback+mic together while playing a sound; run with --ignored"]
fn loopback_recording_with_mic_bridge_captures_played_audio() {
// The exact production path when the mic is on: loopback + mic share a
// MicBridge and the recording must still contain the loopback audio (mic
// silence here just mixes in as 0). Proves the bridge doesn't zero or
// deadlock the recording.
let dir = std::env::temp_dir().join(format!("wa-mixrec-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("audio.wav");
let bridge = MicBridge::shared();
let (loop_tx, _lr) = std::sync::mpsc::sync_channel::<Vec<f32>>(64);
let (mic_tx, _mr) = std::sync::mpsc::sync_channel::<Vec<f32>>(64);
let (ev_tx, _ev) = std::sync::mpsc::sync_channel::<CaptureEvent>(8);
let (ev_tx2, _ev2) = std::sync::mpsc::sync_channel::<CaptureEvent>(8);
let loop_h =
match WasapiCapture.start_loopback_recording(&path, None, loop_tx, ev_tx, bridge.clone())
{
Ok(h) => h,
Err(e) => {
eprintln!("[mixrec] no loopback device: {e} (skipping)");
return;
}
};
let mic_h = WasapiCapture
.start_microphone_recording(None, mic_tx, ev_tx2, bridge)
.ok();
let _ = std::process::Command::new("powershell")
.args([
"-NoProfile",
"-Command",
"(New-Object Media.SoundPlayer 'C:\\Windows\\Media\\Alarm01.wav').PlaySync()",
])
.status();
std::thread::sleep(Duration::from_millis(500));
let summary = WasapiCapture.stop(loop_h).unwrap();
if let Some(m) = mic_h {
let _ = WasapiCapture.stop(m);
}
let raw = std::fs::read(&path).unwrap();
let data_at = raw.windows(4).position(|w| w == b"data").unwrap() + 8;
let peak = raw[data_at..]
.chunks_exact(2)
.map(|c| i16::from_le_bytes([c[0], c[1]]).unsigned_abs())
.max()
.unwrap_or(0);
eprintln!(
"[mixrec] {}Hz {}ch peak i16={peak}",
summary.sample_rate, summary.channels
);
let _ = std::fs::remove_dir_all(&dir);
assert!(peak > 0, "combined loopback+mic recording is silent");
}
#[test]
fn find_render_device_resolves_a_real_enumerated_id() {
let devices = list_render_devices().unwrap();