feat(audio): MicActivity timeline for Phase 3 per-stream 'You' attribution (FR-SPK)
Records, per 100ms of audio.wav frame time, when the mic (the user) was speech-level — captured in the loopback writer where mic and loopback exist separately in the recording's own timebase. you_spans() collapses active windows into merged 'You' ranges. The far side will be diarized with these ranges masked out, so clustering never sees the mic. Struct + span logic land first (tested); capture wiring + stop attribution follow.
This commit is contained in:
@@ -261,6 +261,91 @@ impl VoiceSample {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolution of the mic-activity timeline: one flag per this many ms of audio.
|
||||||
|
const ACTIVITY_WINDOW_MS: u64 = 100;
|
||||||
|
|
||||||
|
/// Phase 3 (FR-SPK): a per-recording timeline of when the microphone — the user,
|
||||||
|
/// "You" — was speaking, in `audio.wav` frame time. Captured in the loopback
|
||||||
|
/// writer (the one place the mic and loopback exist separately in the recording's
|
||||||
|
/// own timebase), so [`you_spans`](Self::you_spans) lines up with what sherpa
|
||||||
|
/// reads back for the far-side pass. The far side is then diarized with these
|
||||||
|
/// ranges masked out, so clustering never sees the mic and can't merge it into a
|
||||||
|
/// speaker. One flag per [`ACTIVITY_WINDOW_MS`]; ~10 flags/s ≈ negligible memory.
|
||||||
|
pub struct MicActivity {
|
||||||
|
/// `audio.wav` (loopback) sample rate; 0 until the loopback stream opens.
|
||||||
|
rate: AtomicU32,
|
||||||
|
/// One bool per window; `true` = the mic was speech-level in that window.
|
||||||
|
windows: Mutex<Vec<bool>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MicActivity {
|
||||||
|
pub fn shared() -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
rate: AtomicU32::new(0),
|
||||||
|
windows: Mutex::new(Vec::new()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The loopback writer publishes its rate so frame positions map to windows.
|
||||||
|
// ponytail: wired into capture_loop + stop in the next Phase 3 slice; the
|
||||||
|
// struct + span logic land first, tested, so the wiring is mechanical.
|
||||||
|
#[allow(dead_code)]
|
||||||
|
fn set_rate(&self, rate: u32) {
|
||||||
|
self.rate.store(rate, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Flag the `audio.wav` frames `[start_frame, start_frame + mic.len())` as
|
||||||
|
/// active when the pulled mic chunk is speech-level. Silence is left `false`
|
||||||
|
/// (the default), so gaps between utterances don't become "You".
|
||||||
|
#[allow(dead_code)]
|
||||||
|
fn mark(&self, start_frame: u64, mic: &[f32]) {
|
||||||
|
let rate = self.rate.load(Ordering::Relaxed) as u64;
|
||||||
|
if rate == 0 || mic.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let rms = (mic.iter().map(|s| s * s).sum::<f32>() / mic.len() as f32).sqrt();
|
||||||
|
if rms < VOICE_ENERGY_FLOOR {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let frames_per_window = (rate * ACTIVITY_WINDOW_MS / 1000).max(1);
|
||||||
|
let w0 = (start_frame / frames_per_window) as usize;
|
||||||
|
// Last window actually containing a sample (mic is non-empty here), so a
|
||||||
|
// chunk ending on a boundary doesn't over-mark the next window.
|
||||||
|
let w1 = ((start_frame + mic.len() as u64 - 1) / frames_per_window) as usize;
|
||||||
|
if let Ok(mut win) = self.windows.lock() {
|
||||||
|
if win.len() <= w1 {
|
||||||
|
win.resize(w1 + 1, false);
|
||||||
|
}
|
||||||
|
win[w0..=w1].fill(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Collapse active windows into merged "You" spans as `(start_ms, end_ms)`.
|
||||||
|
/// Adjacent active windows coalesce; the caller drops sub-`MIN_SPAN_MS` blips.
|
||||||
|
pub fn you_spans(&self) -> Vec<(u64, u64)> {
|
||||||
|
let win = self.windows.lock().map(|w| w.clone()).unwrap_or_default();
|
||||||
|
let mut spans = Vec::new();
|
||||||
|
let mut start: Option<usize> = None;
|
||||||
|
for (i, &active) in win.iter().enumerate() {
|
||||||
|
match (active, start) {
|
||||||
|
(true, None) => start = Some(i),
|
||||||
|
(false, Some(s)) => {
|
||||||
|
spans.push((s as u64 * ACTIVITY_WINDOW_MS, i as u64 * ACTIVITY_WINDOW_MS));
|
||||||
|
start = None;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(s) = start {
|
||||||
|
spans.push((
|
||||||
|
s as u64 * ACTIVITY_WINDOW_MS,
|
||||||
|
win.len() as u64 * ACTIVITY_WINDOW_MS,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
spans
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Number of audio frames in a raw WASAPI byte buffer of the given format.
|
/// Number of audio frames in a raw WASAPI byte buffer of the given format.
|
||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
fn frame_count(bytes: &[u8], format: &WaveFormat) -> usize {
|
fn frame_count(bytes: &[u8], format: &WaveFormat) -> usize {
|
||||||
@@ -1128,6 +1213,25 @@ fn mixer_loop(
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mic_activity_ignores_silence() {
|
||||||
|
let ma = MicActivity::shared();
|
||||||
|
ma.set_rate(16_000);
|
||||||
|
ma.mark(0, &[0.0; 3_200]); // below the energy floor
|
||||||
|
assert!(ma.you_spans().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mic_activity_collapses_speech_into_spans_with_gaps() {
|
||||||
|
let ma = MicActivity::shared();
|
||||||
|
ma.set_rate(16_000); // 1600 frames per 100ms window
|
||||||
|
let speech = vec![0.2f32; 3_200]; // 200ms of speech
|
||||||
|
ma.mark(0, &speech); // frames [0,3200) -> windows 0,1 (ms 0-200)
|
||||||
|
// gap: windows 2,3 (ms 200-400) left silent
|
||||||
|
ma.mark(6_400, &speech); // frames [6400,9600) -> windows 4,5 (ms 400-600)
|
||||||
|
assert_eq!(ma.you_spans(), vec![(0, 200), (400, 600)]);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn voice_sample_skips_silence_and_keeps_speech() {
|
fn voice_sample_skips_silence_and_keeps_speech() {
|
||||||
let vs = VoiceSample::new(16_000);
|
let vs = VoiceSample::new(16_000);
|
||||||
|
|||||||
Reference in New Issue
Block a user