iamdoubz/WhispAssist
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

feat(audio): read_wav_channel_16k + vad_spans; retire MicActivity (FR-SPK)

Browse Source 
Add a channel-selective 16k reader (share the decode with read_wav_mono_16k via
read_wav_16k) and vad_spans (100ms energy-windowed You detection). Remove the
MicActivity live timeline + its capture wiring — the split layout makes the
mic/far-side separation intrinsic to the file, so You is recomputed from the
left channel instead.
This commit is contained in:
iamdoubz committed 2026-07-14 12:16:01 -05:00
1 parent 534f5ca350
commit e287fa5ebe
1 file changed
+89 -130
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/audio/mod.rs
+89 -130
View File
@@ -273,86 +273,11 @@ impl VoiceSample {
}
}
/// Resolution of the mic-activity timeline: one flag per this many ms of audio.
const ACTIVITY_WINDOW_MS: u64 = 100;
/// Phase 3 (FR-SPK): a per-recording timeline of when the microphone — the user,
/// "You" — was speaking, in `audio.wav` frame time. Captured in the loopback
/// writer (the one place the mic and loopback exist separately in the recording's
/// own timebase), so [`you_spans`](Self::you_spans) lines up with what sherpa
/// reads back for the far-side pass. The far side is then diarized with these
/// ranges masked out, so clustering never sees the mic and can't merge it into a
/// speaker. One flag per [`ACTIVITY_WINDOW_MS`]; ~10 flags/s ≈ negligible memory.
pub struct MicActivity {
/// `audio.wav` (loopback) sample rate; 0 until the loopback stream opens.
rate: AtomicU32,
/// One bool per window; `true` = the mic was speech-level in that window.
windows: Mutex<Vec<bool>>,
}
impl MicActivity {
pub fn shared() -> Arc<Self> {
Arc::new(Self {
rate: AtomicU32::new(0),
windows: Mutex::new(Vec::new()),
})
}
/// The loopback writer publishes its rate so frame positions map to windows.
fn set_rate(&self, rate: u32) {
self.rate.store(rate, Ordering::Relaxed);
}
/// Flag the `audio.wav` frames `[start_frame, start_frame + mic.len())` as
/// active when the pulled mic chunk is speech-level. Silence is left `false`
/// (the default), so gaps between utterances don't become "You".
fn mark(&self, start_frame: u64, mic: &[f32]) {
let rate = self.rate.load(Ordering::Relaxed) as u64;
if rate == 0 || mic.is_empty() {
return;
}
let rms = (mic.iter().map(|s| s * s).sum::<f32>() / mic.len() as f32).sqrt();
if rms < VOICE_ENERGY_FLOOR {
return;
}
let frames_per_window = (rate * ACTIVITY_WINDOW_MS / 1000).max(1);
let w0 = (start_frame / frames_per_window) as usize;
// Last window actually containing a sample (mic is non-empty here), so a
// chunk ending on a boundary doesn't over-mark the next window.
let w1 = ((start_frame + mic.len() as u64 - 1) / frames_per_window) as usize;
if let Ok(mut win) = self.windows.lock() {
if win.len() <= w1 {
win.resize(w1 + 1, false);
}
win[w0..=w1].fill(true);
}
}
/// Collapse active windows into merged "You" spans as `(start_ms, end_ms)`.
/// Adjacent active windows coalesce; the caller drops sub-`MIN_SPAN_MS` blips.
pub fn you_spans(&self) -> Vec<(u64, u64)> {
let win = self.windows.lock().map(|w| w.clone()).unwrap_or_default();
let mut spans = Vec::new();
let mut start: Option<usize> = None;
for (i, &active) in win.iter().enumerate() {
match (active, start) {
(true, None) => start = Some(i),
(false, Some(s)) => {
spans.push((s as u64 * ACTIVITY_WINDOW_MS, i as u64 * ACTIVITY_WINDOW_MS));
start = None;
}
_ => {}
}
}
if let Some(s) = start {
spans.push((
s as u64 * ACTIVITY_WINDOW_MS,
win.len() as u64 * ACTIVITY_WINDOW_MS,
));
}
spans
}
}
// MicActivity (the live mic-speech timeline) was retired 2026-07-14 with the
// masked-mono diarization it fed — the split (mic-left/loopback-right) layout
// makes the mic/far-side separation intrinsic to audio.wav, so "You" is
// recomputed from the left channel via `vad_spans`. See the retirement note in
// commands.rs (attribute_split).
/// Number of audio frames in a raw WASAPI byte buffer of the given format.
#[cfg(feature = "audio")]
@@ -387,7 +312,6 @@ impl WasapiCapture {
event_sink: EventSink,
bridge: Option<Arc<MicBridge>>,
voice_sample: Option<Arc<VoiceSample>>,
mic_activity: Option<Arc<MicActivity>>,
split: bool,
emit_level: bool,
) -> Result<CaptureHandle, AudioError> {
@@ -411,7 +335,6 @@ impl WasapiCapture {
&paused_th,
bridge.as_ref(),
voice_sample.as_ref(),
mic_activity.as_ref(),
split,
emit_level,
)
@@ -435,7 +358,6 @@ impl WasapiCapture {
frame_sink: FrameSink,
event_sink: EventSink,
bridge: Arc<MicBridge>,
mic_activity: Option<Arc<MicActivity>>,
) -> Result<CaptureHandle, AudioError> {
self.start_capture(
"wa-audio-capture",
@@ -446,9 +368,8 @@ impl WasapiCapture {
event_sink,
Some(bridge),
None,
mic_activity,
true, // split: mic-enabled loopback records mic-left / loopback-right
true, // emit_level
true, // split: mic-enabled loopback records mic-left / loopback-right
true, // emit_level
)
}
@@ -473,7 +394,6 @@ impl WasapiCapture {
event_sink,
Some(bridge),
voice_sample,
None,
false, // split: mic stream has no writer
true, // emit_level
)
@@ -498,7 +418,6 @@ impl AudioCapture for WasapiCapture {
event_sink,
None,
None,
None,
false, // split: loopback-only / mic stream — no split writer
true, // emit_level
)
@@ -519,7 +438,6 @@ impl AudioCapture for WasapiCapture {
event_sink,
None,
None,
None,
false, // split: loopback-only / mic stream — no split writer
true, // emit_level
)
@@ -694,7 +612,6 @@ fn capture_loop(
paused: &AtomicBool,
bridge: Option<&Arc<MicBridge>>,
voice_sample: Option<&Arc<VoiceSample>>,
mic_activity: Option<&Arc<MicActivity>>,
// FR-SPK: when true, the loopback WAV is stereo L=mic / R=loopback (the mic
// is a separate track, not summed in) so diarization can run on the far side
// alone. Only meaningful for the loopback writer.
@@ -717,9 +634,6 @@ fn capture_loop(
b.rate
.store(session.format.get_samplespersec(), Ordering::Relaxed);
}
if let Some(ma) = mic_activity {
ma.set_rate(session.format.get_samplespersec());
}
}
let mut spec = wav_spec_for(&session.format)?;
if split && is_loopback {
@@ -781,9 +695,6 @@ fn capture_loop(
b.rate
.store(new_session.format.get_samplespersec(), Ordering::Relaxed);
}
if let Some(ma) = mic_activity {
ma.set_rate(new_session.format.get_samplespersec());
}
}
session = new_session;
let _ = event_sink.try_send(CaptureEvent::DeviceChanged {
@@ -818,15 +729,11 @@ fn capture_loop(
let bpf = session.format.get_nchannels() as usize
* (session.format.get_bitspersample() as usize / 8);
let silence = vec![0u8; gap_frames * bpf];
let before = frames_written;
frames_written += if split {
write_wav_split(w, &silence, &session.format, &mic)?
} else {
write_wav_bytes(w, &silence, &session.format, &mic)?
};
if let Some(ma) = mic_activity {
ma.mark(before, &mic);
}
last_write = Instant::now();
}
}
@@ -849,15 +756,11 @@ fn capture_loop(
Some(b) if is_loopback => b.pull(frame_count(&bytes, &session.format)),
_ => Vec::new(),
};
let before = frames_written;
frames_written += if split {
write_wav_split(w, &bytes, &session.format, &mic)?
} else {
write_wav_bytes(w, &bytes, &session.format, &mic)?
};
if let Some(ma) = mic_activity {
ma.mark(before, &mic);
}
}
let mono = decode_mono_f32(&bytes, &session.format)?;
@@ -1148,6 +1051,22 @@ impl Resampler {
/// through hound's size-aware iterator; hound is only used to parse the
/// format/`fmt ` chunk.
pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
read_wav_16k(path, None)
}
/// Like [`read_wav_mono_16k`] but extracts a single channel instead of
/// downmixing — used by the Phase 3.5 split path to diarize the right
/// (loopback) channel and VAD the left (mic) channel (FR-SPK). `channel` is
/// clamped to the file's last channel, so it's safe on a mono/summed file.
pub fn read_wav_channel_16k(path: &Path, channel: usize) -> Result<Vec<f32>, AudioError> {
read_wav_16k(path, Some(channel))
}
/// Shared reader: decrypt (T8.8), reduce each frame to one f32 — averaging all
/// channels (`channel = None`) or picking one (`channel = Some`) — resample to
/// 16kHz. Reads the `data` payload as raw bytes (an unfinalized WAV's size
/// fields read back as 0), so it works on a mid-write / crash-left file (FR-REL-1).
fn read_wav_16k(path: &Path, channel: Option<usize>) -> Result<Vec<f32>, AudioError> {
let raw = std::fs::read(path).map_err(|e| AudioError::Capture(format!("read wav: {e}")))?;
// Decrypt if the recording was sealed at rest (T8.8); plaintext passes through.
let file =
@@ -1157,6 +1076,8 @@ pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
.map_err(|e| AudioError::Capture(format!("open wav: {e}")))?
.spec();
let channels = spec.channels.max(1) as usize;
// Which channel index to pick (clamped), or None to downmix.
let pick = channel.map(|c| c.min(channels - 1));
let data_marker = file
.windows(4)
@@ -1168,22 +1089,14 @@ pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
let mono: Vec<f32> = match (spec.sample_format, spec.bits_per_sample) {
(SampleFormat::Float, 32) => bytes
.chunks_exact(4 * channels)
.map(|frame| {
frame
.chunks_exact(4)
.map(|c| f32::from_le_bytes(c.try_into().unwrap()))
.sum::<f32>()
/ channels as f32
})
.map(|frame| reduce_frame(frame, 4, pick, channels, |c| f32::from_le_bytes(c.try_into().unwrap())))
.collect(),
(SampleFormat::Int, 16) => bytes
.chunks_exact(2 * channels)
.map(|frame| {
frame
.chunks_exact(2)
.map(|c| i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32)
.sum::<f32>()
/ channels as f32
reduce_frame(frame, 2, pick, channels, |c| {
i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32
})
})
.collect(),
(fmt, bits) => {
@@ -1197,6 +1110,56 @@ pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
Ok(resampler.process(&mono))
}
/// Voice-activity "You" spans (FR-SPK): 100ms windows of 16kHz mono audio above
/// [`VOICE_ENERGY_FLOOR`], merged into `(start_ms, end_ms)` ranges. Run on the
/// left (mic) channel of a split recording to attribute segments to "You" —
/// replaces the old live `MicActivity` timeline (recomputed from the file, so
/// stop and reprocess agree).
pub fn vad_spans(samples: &[f32]) -> Vec<(u64, u64)> {
const WINDOW_MS: u64 = 100;
let win = (TARGET_SAMPLE_RATE as u64 * WINDOW_MS / 1000) as usize; // 1600 @16kHz
if win == 0 {
return Vec::new();
}
let mut spans = Vec::new();
let mut start: Option<u64> = None;
let mut w = 0u64;
let mut i = 0usize;
while i < samples.len() {
let chunk = &samples[i..(i + win).min(samples.len())];
let rms = (chunk.iter().map(|s| s * s).sum::<f32>() / chunk.len().max(1) as f32).sqrt();
match (rms >= VOICE_ENERGY_FLOOR, start) {
(true, None) => start = Some(w * WINDOW_MS),
(false, Some(s)) => {
spans.push((s, w * WINDOW_MS));
start = None;
}
_ => {}
}
i += win;
w += 1;
}
if let Some(s) = start {
spans.push((s, w * WINDOW_MS));
}
spans
}
/// Reduce one interleaved frame to a single f32: pick channel `pick` or average
/// all `channels`. `width` is bytes-per-sample; `decode` turns those bytes → f32.
fn reduce_frame(
frame: &[u8],
width: usize,
pick: Option<usize>,
channels: usize,
decode: impl Fn(&[u8]) -> f32,
) -> f32 {
match pick {
Some(c) => decode(&frame[c * width..c * width + width]),
None => frame.chunks_exact(width).map(decode).sum::<f32>() / channels as f32,
}
}
/// Write mono 16kHz `f32` samples as a Float32 WAV — the format
/// [`read_wav_mono_16k`] reads back. Used to persist the mic voiceprint sample
/// next to a retained recording so it can be re-matched on reprocess (FR-SPK).
@@ -1325,22 +1288,19 @@ mod tests {
use super::*;
#[test]
fn mic_activity_ignores_silence() {
let ma = MicActivity::shared();
ma.set_rate(16_000);
ma.mark(0, &[0.0; 3_200]); // below the energy floor
assert!(ma.you_spans().is_empty());
fn vad_spans_ignores_silence() {
// 200ms of 16kHz silence → no "You" spans.
assert!(vad_spans(&[0.0; 3_200]).is_empty());
}
#[test]
fn mic_activity_collapses_speech_into_spans_with_gaps() {
let ma = MicActivity::shared();
ma.set_rate(16_000); // 1600 frames per 100ms window
let speech = vec![0.2f32; 3_200]; // 200ms of speech
ma.mark(0, &speech); // frames [0,3200) -> windows 0,1 (ms 0-200)
// gap: windows 2,3 (ms 200-400) left silent
ma.mark(6_400, &speech); // frames [6400,9600) -> windows 4,5 (ms 400-600)
assert_eq!(ma.you_spans(), vec![(0, 200), (400, 600)]);
fn vad_spans_collapses_speech_into_spans_with_gaps() {
// 16kHz: 1600 samples per 100ms window. Speech in windows 0,1 (0-200ms)
// and 4,5 (400-600ms), silence between.
let mut samples = vec![0.0f32; 9_600]; // 600ms
samples[0..3_200].fill(0.2); // windows 0,1
samples[6_400..9_600].fill(0.2); // windows 4,5
assert_eq!(vad_spans(&samples), vec![(0, 200), (400, 600)]);
}
#[test]
@@ -1741,7 +1701,6 @@ mod tests {
loop_tx,
ev_tx,
bridge.clone(),
None,
) {
Ok(h) => h,
Err(e) => {