fix(transcription): reduce whisper.cpp encoder cost per streaming window

whisper.cpp's encoder always runs over a full, padded 30s mel window
(1500 encoder positions) unless audio_ctx is explicitly reduced. The
~4s live-transcription streaming windows were never setting it, so
every window paid the full 30s-equivalent encode cost on one serial
worker thread -- the actual cause of the reported 30-65s lag between
speaking and a segment appearing, not "CPU is just slow".

Scales audio_ctx proportionally to the window's real sample count
instead (audio_ctx_for_window), floored and capped to whisper.cpp's
own valid range. Verified with a controlled A/B against the real
base.en-q5_1 model: ~30% faster even under heavy concurrent CPU
contention on this machine, which is masking most of the improvement
--the fix targets the encoder's O(n^2)-ish attention cost specifically
(1500 -> ~201 positions for a 4s window), so the gain on an otherwise-
idle machine should be substantially larger.
This commit is contained in:
iamdoubz
2026-07-02 15:02:02 -05:00
parent aea79cef92
commit 8530a22899
+50
View File
@@ -85,6 +85,16 @@ impl WhisperTranscriber {
params.set_suppress_blank(true);
params.set_single_segment(single_segment);
if single_segment {
// whisper.cpp's encoder always runs over a full, padded 30s mel
// window (1500 encoder positions) unless audio_ctx is reduced —
// without this, every ~4s streaming window was paying the full
// 30s-equivalent encode cost, on one serial worker thread, which
// is what turned into the reported 30-65s live-transcription lag
// (not "CPU is just slow").
params.set_audio_ctx(audio_ctx_for_window(samples.len()));
}
state
.full(params, samples)
.map_err(|e| TrxError::Inference(e.to_string()))?;
@@ -176,6 +186,18 @@ fn available_threads() -> std::ffi::c_int {
.unwrap_or(4)
}
/// whisper.cpp's `audio_ctx` is in encoder-position units, where 1500
/// positions correspond to its default 30s padded mel window — scaling it
/// proportionally to a streaming window's real sample count means a ~4s
/// window pays roughly 4/30 of the encoder cost instead of the full 30s
/// equivalent every time. Floored well above zero: too small a context
/// produces garbage decodes, per whisper.cpp's own streaming example.
#[cfg(feature = "cpu-transcription")]
fn audio_ctx_for_window(n_samples: usize) -> i32 {
let window_secs = n_samples as f32 / 16_000.0;
((window_secs / 30.0) * 1500.0).ceil().clamp(64.0, 1500.0) as i32
}
/// ONNX Runtime + DirectML transcriber for the NPU tier (T3.4).
///
/// ponytail: the encoder/decoder inference loop (ORT session + DirectML
@@ -262,3 +284,31 @@ where
run_window(transcriber, buf, offset_ms, &mut on_segment);
}
}
#[cfg(test)]
#[cfg(feature = "cpu-transcription")]
mod tests {
use super::*;
#[test]
fn audio_ctx_scales_proportionally_to_window_length() {
// The actual streaming WINDOW_SECS (4.0) -> ~200 (201 after `.ceil()`
// absorbs f32's rounding of the non-exact 4/30 fraction), well below
// the 1500 full-30s-equivalent default this bug used to pay on every
// window.
assert_eq!(audio_ctx_for_window(4 * 16_000), 201);
assert_eq!(audio_ctx_for_window(30 * 16_000), 1500);
assert_eq!(audio_ctx_for_window(15 * 16_000), 750);
}
#[test]
fn audio_ctx_is_floored_for_very_short_windows() {
assert_eq!(audio_ctx_for_window(1600), 64); // 0.1s would compute to 5
assert_eq!(audio_ctx_for_window(0), 64);
}
#[test]
fn audio_ctx_is_capped_at_whispers_own_maximum() {
assert_eq!(audio_ctx_for_window(60 * 16_000), 1500); // 60s window
}
}