806 lines
33 KiB
Rust
806 lines
33 KiB
Rust
//! Transcription engine (Phase 1 CPU; Phase 3 acceleration). FR-TRX-*.
|
|
//!
|
|
//! Primary engine: whisper-rs (whisper.cpp) for CPU/Vulkan/CUDA. The NPU path is
|
|
//! a second `Transcriber` impl using ONNX Runtime (ort + OpenVINO EP on Intel
|
|
//! NPUs) — same trait, same `TranscriptSegment` output (ADR-0003/0004). Callers
|
|
//! never branch on engine.
|
|
|
|
use crate::models::{BackendId, TranscriptSegment};
|
|
use std::path::Path;
|
|
use std::sync::mpsc::{Receiver, Sender};
|
|
|
|
pub mod languages;
|
|
pub mod models;
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum TrxError {
|
|
#[error("model load failed: {0}")]
|
|
Load(String),
|
|
#[error("inference failed: {0}")]
|
|
Inference(String),
|
|
}
|
|
|
|
/// A chunk of audio (mono f32, 16 kHz) handed to the streaming transcriber.
|
|
pub struct AudioWindow {
|
|
pub samples: Vec<f32>,
|
|
pub offset_ms: u64,
|
|
}
|
|
|
|
/// Where produced segments are delivered (interim then final).
|
|
pub type SegmentSink = Sender<TranscriptSegment>;
|
|
|
|
pub trait Transcriber: Send + Sync {
|
|
/// `language` (T8.7, FR-TRX-4): `None` or `Some("auto")` requests
|
|
/// auto-detection; an explicit ISO-639-1 code (e.g. `"es"`) forces that
|
|
/// language. Engines that can't honor a request (an English-only model,
|
|
/// or an engine with no language selection at all, like the NPU/ONNX
|
|
/// path) resolve it against their own capability at load time rather
|
|
/// than erroring — see `resolve_language` and `effective_language`.
|
|
fn load(model: &Path, backend: BackendId, language: Option<&str>) -> Result<Self, TrxError>
|
|
where
|
|
Self: Sized;
|
|
/// Streaming: emit interim + final segments for a window (FR-TRX-2).
|
|
fn transcribe_stream(&self, audio: AudioWindow, out: SegmentSink) -> Result<(), TrxError>;
|
|
/// Batch: one-shot over a whole file, higher accuracy — also the crash-recovery
|
|
/// path (FR-TRX-3, T2.8): re-run over the working `audio.wav` from scratch.
|
|
fn transcribe_file(&self, wav: &Path) -> Result<Vec<TranscriptSegment>, TrxError>;
|
|
|
|
/// The language this engine is actually configured to decode, resolved
|
|
/// against model capability at `load` time — `None` means auto-detect.
|
|
/// Default: engines with no language selection of their own (the
|
|
/// NPU/ONNX path, whose ONNX artifacts are exported English-only) always
|
|
/// decode English.
|
|
fn effective_language(&self) -> Option<String> {
|
|
Some("en".to_string())
|
|
}
|
|
|
|
/// The language actually used/detected on the most recent decode, if the
|
|
/// engine surfaces one (whisper.cpp does, via `full_lang_id_from_state`).
|
|
/// `None` until at least one decode has completed, or if the engine
|
|
/// doesn't support detection reporting at all.
|
|
fn detected_language(&self) -> Option<String> {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Resolves a requested language against model capability: an English-only
|
|
/// (`.en`) whisper.cpp model can only ever decode English, so an explicit
|
|
/// non-English request is forced to `"en"` (with a warning) rather than
|
|
/// being silently honored into a garbage transcript — the T8.7/FR-TRX-4
|
|
/// "no silent footgun" requirement. `None`/`"auto"` always means
|
|
/// auto-detect, regardless of model, since that's harmless either way
|
|
/// (whisper.cpp itself forces English internally for a non-multilingual
|
|
/// model even when `language` is unset).
|
|
#[cfg(feature = "cpu-transcription")]
|
|
fn resolve_language(requested: Option<&str>, multilingual: bool) -> Option<String> {
|
|
match requested {
|
|
None => None,
|
|
Some(l) if l.eq_ignore_ascii_case("auto") => None,
|
|
Some(l) if l.eq_ignore_ascii_case("en") => Some("en".to_string()),
|
|
Some(l) if !multilingual => {
|
|
tracing::warn!(
|
|
"language '{l}' requested but the loaded model is English-only; forcing 'en'"
|
|
);
|
|
Some("en".to_string())
|
|
}
|
|
Some(l) => Some(l.to_string()),
|
|
}
|
|
}
|
|
|
|
/// Segments whisper.cpp itself flags as more likely silence than speech are
|
|
/// dropped rather than emitted (see `run_full`) — whisper.cpp's CLI ships
|
|
/// this same 0.6 default for `--no-speech-thold`.
|
|
#[cfg(feature = "cpu-transcription")]
|
|
const NO_SPEECH_THRESHOLD: f32 = 0.6;
|
|
|
|
#[cfg(feature = "cpu-transcription")]
|
|
fn is_likely_speech(no_speech_probability: f32) -> bool {
|
|
no_speech_probability <= NO_SPEECH_THRESHOLD
|
|
}
|
|
|
|
/// whisper.cpp-backed transcriber (CPU baseline; GPU via Cargo features).
|
|
#[cfg(feature = "cpu-transcription")]
|
|
pub struct WhisperTranscriber {
|
|
ctx: whisper_rs::WhisperContext,
|
|
next_id: std::sync::atomic::AtomicU64,
|
|
/// What gets passed to `FullParams::set_language` on every decode —
|
|
/// resolved once at `load` (see `resolve_language`), not re-resolved per
|
|
/// window/file, since a whole recording session uses one language.
|
|
configured_language: Option<String>,
|
|
/// The language whisper.cpp actually used on the most recent `full()`
|
|
/// call (`whisper_full_lang_id_from_state`), updated after every decode
|
|
/// so "auto" mode has something concrete to persist (T8.7, FR-TRX-4).
|
|
last_detected_language: std::sync::Mutex<Option<String>>,
|
|
}
|
|
|
|
#[cfg(feature = "cpu-transcription")]
|
|
impl WhisperTranscriber {
|
|
/// Shared decode path for both streaming windows and batch files.
|
|
///
|
|
/// `single_segment` forces whisper.cpp to treat the whole buffer as one
|
|
/// segment — needed for the short (~4s) streaming windows (see
|
|
/// `transcribe_stream`'s doc comment for why), but wrong for batch/file
|
|
/// transcription: a multi-minute recording should get whisper's normal
|
|
/// speech-boundary-driven multi-segment output, and its internal 30s-chunk
|
|
/// seek loop handles audio longer than one chunk on its own.
|
|
fn run_full(
|
|
&self,
|
|
samples: &[f32],
|
|
offset_ms: u64,
|
|
single_segment: bool,
|
|
) -> Result<Vec<TranscriptSegment>, TrxError> {
|
|
// whisper.cpp hangs (not errors) on an empty buffer — verified against
|
|
// a crash-recovered WAV whose header hadn't been finalized. Belt and
|
|
// suspenders alongside `read_wav_mono_16k` reading raw bytes directly.
|
|
if samples.is_empty() {
|
|
return Ok(Vec::new());
|
|
}
|
|
|
|
let mut state = self
|
|
.ctx
|
|
.create_state()
|
|
.map_err(|e| TrxError::Inference(e.to_string()))?;
|
|
|
|
// best_of: 1 — fastest greedy decode, matching the CPU/typical-laptop
|
|
// baseline (NFR-PERF-3).
|
|
let mut params =
|
|
whisper_rs::FullParams::new(whisper_rs::SamplingStrategy::Greedy { best_of: 1 });
|
|
params.set_n_threads(available_threads());
|
|
params.set_translate(false);
|
|
params.set_print_progress(false);
|
|
params.set_print_realtime(false);
|
|
params.set_print_timestamps(false);
|
|
params.set_suppress_blank(true);
|
|
params.set_single_segment(single_segment);
|
|
// T8.7/FR-TRX-4: `None` here means auto-detect, matching
|
|
// `configured_language`'s resolved meaning (see `resolve_language`).
|
|
params.set_language(self.configured_language.as_deref());
|
|
|
|
if single_segment {
|
|
// whisper.cpp's encoder always runs over a full, padded 30s mel
|
|
// window (1500 encoder positions) unless audio_ctx is reduced —
|
|
// without this, every ~4s streaming window was paying the full
|
|
// 30s-equivalent encode cost, on one serial worker thread, which
|
|
// is what turned into the reported 30-65s live-transcription lag
|
|
// (not "CPU is just slow").
|
|
params.set_audio_ctx(audio_ctx_for_window(samples.len()));
|
|
}
|
|
|
|
state
|
|
.full(params, samples)
|
|
.map_err(|e| TrxError::Inference(e.to_string()))?;
|
|
|
|
// T8.7/FR-TRX-4: record whatever language whisper.cpp actually used
|
|
// for this decode (explicit request or auto-detected) so "auto" mode
|
|
// has a concrete value to persist per meeting. Best-effort — an
|
|
// unrecognized lang id (or a poisoned mutex) just leaves the last
|
|
// known value in place rather than failing the transcription.
|
|
let lang_id = state.full_lang_id_from_state();
|
|
if let Some(code) = whisper_rs::get_lang_str(lang_id) {
|
|
if let Ok(mut last) = self.last_detected_language.lock() {
|
|
*last = Some(code.to_string());
|
|
}
|
|
}
|
|
|
|
// A forced single_segment's reported end_timestamp() reflects
|
|
// whisper.cpp's internal 30s-padded mel frame, not the real window
|
|
// length — confirmed even with `duration_ms` set, so don't trust it.
|
|
// We already know the window's true length from `samples`, which is
|
|
// the more reliable source for `single_segment` windows; whisper's own
|
|
// multi-segment timestamps (batch/`transcribe_file`) stay authoritative.
|
|
let window_end_ms = offset_ms + (samples.len() as u64 * 1000) / 16_000;
|
|
|
|
let mut out = Vec::new();
|
|
for seg in state.as_iter() {
|
|
let text = seg.to_str_lossy().unwrap_or_default().trim().to_string();
|
|
if text.is_empty() {
|
|
continue;
|
|
}
|
|
// whisper.cpp's own `no_speech_thold` gate is a no-op (unimplemented
|
|
// upstream as of whisper-rs 0.16 / whisper.cpp v1.3.0+), so silent/
|
|
// near-silent windows still decode — and greedy short-window decode
|
|
// reliably hallucinates a short filler word ("you", "Thank you.")
|
|
// instead of emitting nothing. Drop those ourselves: matches
|
|
// whisper.cpp's own CLI default threshold for "this was silence".
|
|
if !is_likely_speech(seg.no_speech_probability()) {
|
|
continue;
|
|
}
|
|
// Whisper timestamps are centiseconds (10ms units).
|
|
let start_ms = offset_ms + seg.start_timestamp().max(0) as u64 * 10;
|
|
let end_ms = if single_segment {
|
|
window_end_ms.max(start_ms)
|
|
} else {
|
|
offset_ms + seg.end_timestamp().max(0) as u64 * 10
|
|
};
|
|
out.push(TranscriptSegment {
|
|
id: self
|
|
.next_id
|
|
.fetch_add(1, std::sync::atomic::Ordering::SeqCst),
|
|
start_ms,
|
|
end_ms,
|
|
// Diarization lands in Phase 4; every segment is provisionally "S1" until then.
|
|
speaker: "S1".to_string(),
|
|
text,
|
|
confidence: Some((1.0 - seg.no_speech_probability()).clamp(0.0, 1.0)),
|
|
interim: false,
|
|
});
|
|
}
|
|
Ok(out)
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "cpu-transcription")]
|
|
impl Transcriber for WhisperTranscriber {
|
|
/// `backend` picks GPU offload at runtime (T3.2): whisper.cpp only has GPU
|
|
/// support at all when built with the `cuda`/`vulkan` Cargo feature (T3.3),
|
|
/// so `use_gpu` on a CPU-only build is a harmless no-op — this stays a
|
|
/// single code path either way rather than branching on which features
|
|
/// were compiled in.
|
|
fn load(model: &Path, backend: BackendId, language: Option<&str>) -> Result<Self, TrxError> {
|
|
let params = whisper_rs::WhisperContextParameters {
|
|
use_gpu: !matches!(backend, BackendId::Cpu),
|
|
..Default::default()
|
|
};
|
|
let ctx = whisper_rs::WhisperContext::new_with_params(model, params)
|
|
.map_err(|e| TrxError::Load(e.to_string()))?;
|
|
let configured_language = resolve_language(language, ctx.is_multilingual());
|
|
Ok(Self {
|
|
ctx,
|
|
next_id: std::sync::atomic::AtomicU64::new(0),
|
|
configured_language,
|
|
last_detected_language: std::sync::Mutex::new(None),
|
|
})
|
|
}
|
|
|
|
fn transcribe_stream(&self, audio: AudioWindow, out: SegmentSink) -> Result<(), TrxError> {
|
|
// Each call already gets exactly one fixed-size window (not a full 30s
|
|
// clip); without `single_segment: true`, whisper.cpp's segment-boundary
|
|
// heuristic hits "single timestamp ending" on short buffers and drops
|
|
// the whole window even when decoding succeeded (verified against real
|
|
// audio — see whisper.cpp's own `stream` example, which sets this for
|
|
// the same reason).
|
|
for segment in self.run_full(&audio.samples, audio.offset_ms, true)? {
|
|
if out.send(segment).is_err() {
|
|
break; // receiver gone (meeting stopped) — nothing left to do
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn transcribe_file(&self, wav: &Path) -> Result<Vec<TranscriptSegment>, TrxError> {
|
|
let samples =
|
|
crate::audio::read_wav_mono_16k(wav).map_err(|e| TrxError::Load(e.to_string()))?;
|
|
self.run_full(&samples, 0, false)
|
|
}
|
|
|
|
fn effective_language(&self) -> Option<String> {
|
|
self.configured_language.clone()
|
|
}
|
|
|
|
fn detected_language(&self) -> Option<String> {
|
|
self.last_detected_language
|
|
.lock()
|
|
.ok()
|
|
.and_then(|g| g.clone())
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "cpu-transcription")]
|
|
fn available_threads() -> std::ffi::c_int {
|
|
std::thread::available_parallelism()
|
|
.map(|n| n.get() as std::ffi::c_int)
|
|
.unwrap_or(4)
|
|
}
|
|
|
|
/// whisper.cpp's `audio_ctx` is in encoder-position units, where 1500
|
|
/// positions correspond to its default 30s padded mel window — scaling it
|
|
/// proportionally to a streaming window's real sample count means a ~4s
|
|
/// window pays roughly 4/30 of the encoder cost instead of the full 30s
|
|
/// equivalent every time. Floored well above zero: too small a context
|
|
/// produces garbage decodes, per whisper.cpp's own streaming example.
|
|
#[cfg(feature = "cpu-transcription")]
|
|
fn audio_ctx_for_window(n_samples: usize) -> i32 {
|
|
let window_secs = n_samples as f32 / 16_000.0;
|
|
((window_secs / 30.0) * 1500.0).ceil().clamp(64.0, 1500.0) as i32
|
|
}
|
|
|
|
// ONNX tier: Whisper ONNX via ONNX Runtime on the Intel NPU (OpenVINO, T3.4) or
|
|
// a DX12 GPU (DirectML — the non-Vulkan AMD/Intel path, P2). One
|
|
// `OnnxTranscriber` implementing the same `Transcriber` trait; see `npu.rs`.
|
|
// Gated on the `npu` feature so the CPU-only build never pulls `ort`.
|
|
#[cfg(feature = "npu")]
|
|
pub mod mel;
|
|
#[cfg(feature = "npu")]
|
|
pub mod npu;
|
|
#[cfg(feature = "npu")]
|
|
pub mod onnx_models;
|
|
#[cfg(feature = "npu")]
|
|
pub use npu::OnnxTranscriber;
|
|
|
|
const STREAM_SAMPLE_RATE: usize = 16_000;
|
|
|
|
/// Cadence/length knobs for the live streaming worker (`Streamer`). Tuned for a
|
|
/// fluid transcript that shows words ~1s after they're spoken and breaks lines
|
|
/// at natural pauses rather than on a fixed clock.
|
|
#[derive(Debug, Clone, Copy)]
|
|
pub struct StreamTuning {
|
|
/// How often the growing window is re-decoded (and the interim line
|
|
/// refreshed). Lower = more responsive, but more CPU (each decode re-runs
|
|
/// whisper over the whole in-progress window).
|
|
pub step_ms: u64,
|
|
/// Hard cap on an uncommitted window: once the current line reaches this
|
|
/// without a natural pause, it's force-committed so the window (and its
|
|
/// per-step decode cost) can't grow without bound.
|
|
pub max_window_ms: u64,
|
|
/// Don't commit a line shorter than this on a detected pause — avoids
|
|
/// chopping a brief hesitation into its own one-word line.
|
|
pub min_commit_ms: u64,
|
|
}
|
|
|
|
impl StreamTuning {
|
|
/// `low_overhead` doubles the decode step (halving CPU) at the cost of a
|
|
/// slightly less immediate transcript — matches the Settings "low overhead"
|
|
/// preset (CPU + smallest model, for battery/background use).
|
|
pub fn new(low_overhead: bool) -> Self {
|
|
Self {
|
|
step_ms: if low_overhead { 2000 } else { 1000 },
|
|
max_window_ms: 12_000,
|
|
min_commit_ms: if low_overhead { 2000 } else { 1500 },
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Default for StreamTuning {
|
|
fn default() -> Self {
|
|
Self::new(false)
|
|
}
|
|
}
|
|
|
|
/// Incremental live-transcript state machine, independent of any transcription
|
|
/// engine (a `decode: &[f32] -> String` closure is injected) so its
|
|
/// commit/interim logic is unit-testable without whisper.
|
|
///
|
|
/// The model: keep one growing "uncommitted" window of audio. Every `step_ms`,
|
|
/// re-decode the whole window and emit it as an **interim** segment with a
|
|
/// stable id — so the current line grows in place (fluid, low-latency) instead
|
|
/// of popping in whole every few seconds. When the decoded text stops changing
|
|
/// for a step (the speaker paused, so the extra audio was silence) the line is
|
|
/// **committed** (`interim = false`, same id) and a fresh window/line begins —
|
|
/// so lines break at natural sentence pauses, not on a fixed 4s clock. A
|
|
/// `max_window_ms` backstop force-commits a pause-free monologue.
|
|
///
|
|
/// whisper.cpp isn't a true streaming recognizer (it re-decodes from scratch),
|
|
/// so this trades CPU — the growing window is re-decoded every step — for a
|
|
/// natural-looking transcript. `audio_ctx_for_window` keeps each decode's cost
|
|
/// proportional to the window length rather than a full 30s encode, and
|
|
/// committing on pauses keeps the window short for conversational speech.
|
|
pub struct Streamer {
|
|
step_len: usize,
|
|
max_len: usize,
|
|
min_commit_len: usize,
|
|
window: Vec<f32>,
|
|
committed_offset_ms: u64,
|
|
since_last_decode: usize,
|
|
next_id: u64,
|
|
last_text: String,
|
|
}
|
|
|
|
impl Streamer {
|
|
pub fn new(tuning: StreamTuning) -> Self {
|
|
let per_ms = STREAM_SAMPLE_RATE / 1000;
|
|
Self {
|
|
step_len: tuning.step_ms as usize * per_ms,
|
|
max_len: tuning.max_window_ms as usize * per_ms,
|
|
min_commit_len: tuning.min_commit_ms as usize * per_ms,
|
|
window: Vec::new(),
|
|
committed_offset_ms: 0,
|
|
since_last_decode: 0,
|
|
next_id: 0,
|
|
last_text: String::new(),
|
|
}
|
|
}
|
|
|
|
/// Append newly captured audio; decode + emit once a step's worth has
|
|
/// accumulated. `decode(samples, offset_ms)` returns the transcript of the
|
|
/// window so far (empty for silence).
|
|
pub fn feed<D, F>(&mut self, chunk: &[f32], decode: &D, on_segment: &mut F)
|
|
where
|
|
D: Fn(&[f32], u64) -> String,
|
|
F: FnMut(TranscriptSegment),
|
|
{
|
|
self.window.extend_from_slice(chunk);
|
|
self.since_last_decode += chunk.len();
|
|
if self.since_last_decode >= self.step_len {
|
|
self.since_last_decode = 0;
|
|
self.tick(false, decode, on_segment);
|
|
}
|
|
}
|
|
|
|
/// Commit whatever's in flight — called once when capture stops so the last
|
|
/// in-progress line is finalized rather than left interim.
|
|
pub fn flush<D, F>(&mut self, decode: &D, on_segment: &mut F)
|
|
where
|
|
D: Fn(&[f32], u64) -> String,
|
|
F: FnMut(TranscriptSegment),
|
|
{
|
|
self.tick(true, decode, on_segment);
|
|
}
|
|
|
|
fn tick<D, F>(&mut self, force: bool, decode: &D, on_segment: &mut F)
|
|
where
|
|
D: Fn(&[f32], u64) -> String,
|
|
F: FnMut(TranscriptSegment),
|
|
{
|
|
if self.window.is_empty() {
|
|
return;
|
|
}
|
|
let window_ms = (self.window.len() as u64 * 1000) / STREAM_SAMPLE_RATE as u64;
|
|
let text = decode(&self.window, self.committed_offset_ms);
|
|
let over_max = self.window.len() >= self.max_len;
|
|
|
|
// Nothing recognized yet (leading/standalone silence): don't show an
|
|
// empty line, but still drop the buffer once it's grown too big so we
|
|
// aren't re-decoding a long silence every step.
|
|
if text.is_empty() {
|
|
if over_max || force {
|
|
self.reset(window_ms, false);
|
|
}
|
|
return;
|
|
}
|
|
|
|
let stable = text == self.last_text;
|
|
let commit = force || over_max || (stable && self.window.len() >= self.min_commit_len);
|
|
on_segment(TranscriptSegment {
|
|
id: self.next_id,
|
|
start_ms: self.committed_offset_ms,
|
|
end_ms: self.committed_offset_ms + window_ms,
|
|
// Provisional speaker; the post-stop diarization pass reassigns.
|
|
speaker: "S1".to_string(),
|
|
text: text.clone(),
|
|
confidence: None,
|
|
interim: !commit,
|
|
});
|
|
if commit {
|
|
self.reset(window_ms, true);
|
|
} else {
|
|
self.last_text = text;
|
|
}
|
|
}
|
|
|
|
/// Start a fresh window/line after a commit (`new_line`) or after dropping
|
|
/// leading silence (`!new_line`, which reuses the id since nothing was
|
|
/// emitted for it).
|
|
fn reset(&mut self, window_ms: u64, new_line: bool) {
|
|
self.committed_offset_ms += window_ms;
|
|
self.window.clear();
|
|
self.last_text.clear();
|
|
self.since_last_decode = 0;
|
|
if new_line {
|
|
self.next_id += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Live streaming worker (FR-TRX-2): drives a `Streamer` from the `audio`
|
|
/// service's 16kHz-mono frames, decoding each growing window with `transcriber`
|
|
/// and forwarding interim-then-final segments to `on_segment` (a Tauri emit).
|
|
///
|
|
/// `transcribe_stream` is synchronous for whisper.cpp — by the time it returns,
|
|
/// every segment it produced is already in the per-call channel — so this
|
|
/// drains it inline and joins the text into the window's transcript for the
|
|
/// `Streamer` to diff/commit.
|
|
#[cfg(feature = "cpu-transcription")]
|
|
pub fn run_streaming_worker<T, F>(
|
|
transcriber: &T,
|
|
frame_rx: Receiver<Vec<f32>>,
|
|
tuning: StreamTuning,
|
|
mut on_segment: F,
|
|
) where
|
|
// `?Sized` so the dispatcher can hand us a `&dyn Transcriber` (whisper.cpp
|
|
// or the NPU engine, chosen at runtime) rather than a concrete type.
|
|
T: Transcriber + ?Sized,
|
|
F: FnMut(TranscriptSegment),
|
|
{
|
|
let decode = |samples: &[f32], offset_ms: u64| -> String {
|
|
let (tx, rx) = std::sync::mpsc::channel();
|
|
let window = AudioWindow {
|
|
samples: samples.to_vec(),
|
|
offset_ms,
|
|
};
|
|
if let Err(e) = transcriber.transcribe_stream(window, tx) {
|
|
tracing::warn!("transcription window failed: {e}");
|
|
return String::new();
|
|
}
|
|
let mut parts = Vec::new();
|
|
while let Ok(segment) = rx.try_recv() {
|
|
let t = segment.text.trim();
|
|
if !t.is_empty() {
|
|
parts.push(t.to_string());
|
|
}
|
|
}
|
|
parts.join(" ")
|
|
};
|
|
|
|
let mut streamer = Streamer::new(tuning);
|
|
while let Ok(chunk) = frame_rx.recv() {
|
|
streamer.feed(&chunk, &decode, &mut on_segment);
|
|
}
|
|
streamer.flush(&decode, &mut on_segment);
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[cfg(feature = "cpu-transcription")]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
// Collect a step's emitted segments into `out` — the pushing closure lives
|
|
// and dies inside the call, so `out` is free to read in the asserts after
|
|
// (a single long-lived `on` closure would keep `out` mutably borrowed).
|
|
fn feed_into<D: Fn(&[f32], u64) -> String>(
|
|
s: &mut Streamer,
|
|
chunk: &[f32],
|
|
decode: &D,
|
|
out: &mut Vec<TranscriptSegment>,
|
|
) {
|
|
s.feed(chunk, decode, &mut |seg| out.push(seg));
|
|
}
|
|
|
|
fn flush_into<D: Fn(&[f32], u64) -> String>(
|
|
s: &mut Streamer,
|
|
decode: &D,
|
|
out: &mut Vec<TranscriptSegment>,
|
|
) {
|
|
s.flush(decode, &mut |seg| out.push(seg));
|
|
}
|
|
|
|
#[test]
|
|
fn streamer_grows_interim_then_commits_on_a_pause() {
|
|
// Text grows for two steps, then repeats (the speaker paused, so the
|
|
// extra second was silence) → the line commits.
|
|
let tuning = StreamTuning {
|
|
step_ms: 1000,
|
|
max_window_ms: 12_000,
|
|
min_commit_ms: 1500,
|
|
};
|
|
let mut s = Streamer::new(tuning);
|
|
let script = std::cell::Cell::new(0usize);
|
|
let texts = ["hello", "hello world", "hello world"];
|
|
let decode = |_: &[f32], _off: u64| -> String {
|
|
let i = script.get();
|
|
script.set(i + 1);
|
|
texts.get(i).copied().unwrap_or("hello world").to_string()
|
|
};
|
|
let mut out: Vec<TranscriptSegment> = Vec::new();
|
|
let step = vec![0.0f32; 16_000]; // exactly one 1s step
|
|
|
|
feed_into(&mut s, &step, &decode, &mut out); // "hello" — interim
|
|
feed_into(&mut s, &step, &decode, &mut out); // "hello world" — interim
|
|
feed_into(&mut s, &step, &decode, &mut out); // stable + past min_commit → commit
|
|
|
|
assert_eq!(out.len(), 3);
|
|
assert!(out[0].interim && out[0].text == "hello");
|
|
assert!(out[1].interim && out[1].text == "hello world");
|
|
assert!(!out[2].interim && out[2].text == "hello world");
|
|
assert_eq!(out[0].id, out[2].id, "same line id until it commits");
|
|
|
|
// A new line after the commit uses a fresh id and a later offset.
|
|
let d2 = |_: &[f32], _o: u64| "next sentence".to_string();
|
|
feed_into(&mut s, &step, &d2, &mut out);
|
|
assert!(out[3].interim && out[3].text == "next sentence");
|
|
assert_ne!(out[3].id, out[2].id);
|
|
assert!(
|
|
out[3].start_ms >= 3000,
|
|
"starts after the 3 committed steps"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn streamer_shows_no_line_for_pure_silence() {
|
|
let mut s = Streamer::new(StreamTuning::new(false));
|
|
let decode = |_: &[f32], _o: u64| String::new();
|
|
let mut out: Vec<TranscriptSegment> = Vec::new();
|
|
let step = vec![0.0f32; 16_000];
|
|
for _ in 0..5 {
|
|
feed_into(&mut s, &step, &decode, &mut out);
|
|
}
|
|
assert!(out.is_empty(), "silence must not emit an empty line");
|
|
}
|
|
|
|
#[test]
|
|
fn streamer_flush_commits_the_in_progress_line() {
|
|
let mut s = Streamer::new(StreamTuning::new(false));
|
|
let decode = |_: &[f32], _o: u64| "partial".to_string();
|
|
let mut out: Vec<TranscriptSegment> = Vec::new();
|
|
feed_into(&mut s, &vec![0.0f32; 16_000], &decode, &mut out);
|
|
assert!(out.last().unwrap().interim, "still growing before flush");
|
|
flush_into(&mut s, &decode, &mut out);
|
|
let last = out.last().unwrap();
|
|
assert!(
|
|
!last.interim && last.text == "partial",
|
|
"flush finalizes it"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn streamer_force_commits_a_pause_free_monologue_at_max() {
|
|
// Text never repeats (continuous speech), so only the max-window
|
|
// backstop can commit it — otherwise the window (and decode cost) grows
|
|
// without bound.
|
|
let tuning = StreamTuning {
|
|
step_ms: 1000,
|
|
max_window_ms: 3000,
|
|
min_commit_ms: 1500,
|
|
};
|
|
let mut s = Streamer::new(tuning);
|
|
let n = std::cell::Cell::new(0usize);
|
|
let decode = |_: &[f32], _o: u64| {
|
|
let i = n.get();
|
|
n.set(i + 1);
|
|
format!("word{i}")
|
|
};
|
|
let mut out: Vec<TranscriptSegment> = Vec::new();
|
|
let step = vec![0.0f32; 16_000];
|
|
feed_into(&mut s, &step, &decode, &mut out); // 1s
|
|
feed_into(&mut s, &step, &decode, &mut out); // 2s
|
|
feed_into(&mut s, &step, &decode, &mut out); // 3s == max → force commit
|
|
assert!(!out.last().unwrap().interim);
|
|
}
|
|
|
|
#[test]
|
|
fn audio_ctx_scales_proportionally_to_window_length() {
|
|
// The actual streaming WINDOW_SECS (4.0) -> ~200 (201 after `.ceil()`
|
|
// absorbs f32's rounding of the non-exact 4/30 fraction), well below
|
|
// the 1500 full-30s-equivalent default this bug used to pay on every
|
|
// window.
|
|
assert_eq!(audio_ctx_for_window(4 * 16_000), 201);
|
|
assert_eq!(audio_ctx_for_window(30 * 16_000), 1500);
|
|
assert_eq!(audio_ctx_for_window(15 * 16_000), 750);
|
|
}
|
|
|
|
#[test]
|
|
fn audio_ctx_is_floored_for_very_short_windows() {
|
|
assert_eq!(audio_ctx_for_window(1600), 64); // 0.1s would compute to 5
|
|
assert_eq!(audio_ctx_for_window(0), 64);
|
|
}
|
|
|
|
#[test]
|
|
fn audio_ctx_is_capped_at_whispers_own_maximum() {
|
|
assert_eq!(audio_ctx_for_window(60 * 16_000), 1500); // 60s window
|
|
}
|
|
|
|
#[test]
|
|
fn is_likely_speech_drops_high_no_speech_probability_windows() {
|
|
// Silence/near-silence: whisper.cpp's own greedy hallucination case
|
|
// ("you", "Thank you.") on an otherwise-quiet window.
|
|
assert!(!is_likely_speech(0.9));
|
|
assert!(!is_likely_speech(NO_SPEECH_THRESHOLD + 0.01));
|
|
// Confident speech kept, including right at the threshold.
|
|
assert!(is_likely_speech(0.0));
|
|
assert!(is_likely_speech(NO_SPEECH_THRESHOLD));
|
|
}
|
|
|
|
/// GPU spike (opt-in): times whisper.cpp on a chosen backend against a real
|
|
/// model + wav. With `--features vulkan` and `WA_BACKEND=intel` (or nvidia/amd)
|
|
/// whisper.cpp offloads to the GPU; `WA_BACKEND=cpu` is the baseline. Run:
|
|
/// WA_WHISPER_MODEL=…ggml-base.en-q5_1.bin WA_TEST_WAV=…tts_test.wav \
|
|
/// cargo test --features vulkan gpu_transcribes -- --ignored --nocapture
|
|
#[test]
|
|
#[ignore = "requires a whisper model + wav; GPU/CPU timing spike"]
|
|
fn gpu_transcribes_and_times() {
|
|
let model = std::env::var("WA_WHISPER_MODEL").expect("set WA_WHISPER_MODEL");
|
|
let wav = std::env::var("WA_TEST_WAV").expect("set WA_TEST_WAV");
|
|
let backend = match std::env::var("WA_BACKEND").as_deref() {
|
|
Ok("cpu") => BackendId::Cpu,
|
|
Ok("nvidia") => BackendId::Nvidia,
|
|
Ok("amd") => BackendId::Amd,
|
|
_ => BackendId::Intel, // use_gpu = true for any non-CPU backend
|
|
};
|
|
let t0 = std::time::Instant::now();
|
|
let transcriber = WhisperTranscriber::load(Path::new(&model), backend, None).expect("load");
|
|
let load_ms = t0.elapsed().as_millis();
|
|
let t1 = std::time::Instant::now();
|
|
let segments = transcriber
|
|
.transcribe_file(Path::new(&wav))
|
|
.expect("transcribe");
|
|
let infer_ms = t1.elapsed().as_millis();
|
|
let text = segments
|
|
.iter()
|
|
.map(|s| s.text.as_str())
|
|
.collect::<Vec<_>>()
|
|
.join(" ");
|
|
eprintln!("[spike] backend={backend:?} load={load_ms}ms infer={infer_ms}ms text={text:?}");
|
|
assert!(!text.trim().is_empty(), "transcript was empty");
|
|
}
|
|
|
|
#[test]
|
|
fn resolve_language_auto_and_none_both_mean_auto_detect() {
|
|
assert_eq!(resolve_language(None, true), None);
|
|
assert_eq!(resolve_language(Some("auto"), true), None);
|
|
assert_eq!(resolve_language(Some("AUTO"), true), None);
|
|
}
|
|
|
|
#[test]
|
|
fn resolve_language_explicit_on_multilingual_model_passes_through() {
|
|
assert_eq!(resolve_language(Some("es"), true), Some("es".to_string()));
|
|
assert_eq!(resolve_language(Some("fr"), true), Some("fr".to_string()));
|
|
}
|
|
|
|
#[test]
|
|
fn resolve_language_forces_english_on_english_only_model() {
|
|
// The footgun guard (T8.7/FR-TRX-4): an English-only model can't
|
|
// honor a non-English request, so it's forced to "en" rather than
|
|
// silently producing garbage.
|
|
assert_eq!(resolve_language(Some("es"), false), Some("en".to_string()));
|
|
assert_eq!(resolve_language(Some("EN"), false), Some("en".to_string()));
|
|
// Auto is still allowed on an English-only model — harmless, since
|
|
// whisper.cpp forces English internally for it either way.
|
|
assert_eq!(resolve_language(None, false), None);
|
|
assert_eq!(resolve_language(Some("auto"), false), None);
|
|
}
|
|
|
|
/// Multilingual decode acceptance spike (opt-in, mirrors
|
|
/// `gpu_transcribes_and_times`): loads a *multilingual* model with an
|
|
/// explicit non-English `language` and confirms it decodes non-empty
|
|
/// text without being forced to English. Needs a real multilingual ggml
|
|
/// model + a non-English wav, neither of which are fetched by CI/this
|
|
/// sandbox — run manually:
|
|
/// WA_WHISPER_MODEL=…ggml-small-q5_1.bin WA_TEST_WAV=…spanish.wav \
|
|
/// WA_TEST_LANGUAGE=es cargo test multilingual_model_decodes_requested_language \
|
|
/// -- --ignored --nocapture
|
|
#[test]
|
|
#[ignore = "requires a multilingual whisper model + non-English wav; run manually"]
|
|
fn multilingual_model_decodes_requested_language() {
|
|
let model = std::env::var("WA_WHISPER_MODEL").expect("set WA_WHISPER_MODEL");
|
|
let wav = std::env::var("WA_TEST_WAV").expect("set WA_TEST_WAV");
|
|
let language = std::env::var("WA_TEST_LANGUAGE").unwrap_or_else(|_| "es".to_string());
|
|
|
|
let transcriber =
|
|
WhisperTranscriber::load(Path::new(&model), BackendId::Cpu, Some(&language))
|
|
.expect("load multilingual model");
|
|
assert_eq!(transcriber.effective_language(), Some(language.clone()));
|
|
|
|
let segments = transcriber
|
|
.transcribe_file(Path::new(&wav))
|
|
.expect("transcribe");
|
|
let text = segments
|
|
.iter()
|
|
.map(|s| s.text.as_str())
|
|
.collect::<Vec<_>>()
|
|
.join(" ");
|
|
eprintln!("[spike] language={language} text={text:?}");
|
|
assert!(!text.trim().is_empty(), "transcript was empty");
|
|
// whisper.cpp reports back whichever language it actually decoded in.
|
|
assert_eq!(
|
|
transcriber.detected_language().as_deref(),
|
|
Some(language.as_str())
|
|
);
|
|
}
|
|
|
|
/// "Auto" acceptance spike (opt-in): confirms auto-detection actually
|
|
/// runs (no `language` forced) and surfaces a detected language after
|
|
/// decode. Same manual-only posture as the spike above.
|
|
#[test]
|
|
#[ignore = "requires a multilingual whisper model + wav; run manually"]
|
|
fn auto_mode_detects_a_language() {
|
|
let model = std::env::var("WA_WHISPER_MODEL").expect("set WA_WHISPER_MODEL");
|
|
let wav = std::env::var("WA_TEST_WAV").expect("set WA_TEST_WAV");
|
|
|
|
let transcriber = WhisperTranscriber::load(Path::new(&model), BackendId::Cpu, None)
|
|
.expect("load multilingual model");
|
|
assert_eq!(
|
|
transcriber.effective_language(),
|
|
None,
|
|
"auto should stay unset"
|
|
);
|
|
assert_eq!(transcriber.detected_language(), None, "nothing decoded yet");
|
|
|
|
transcriber
|
|
.transcribe_file(Path::new(&wav))
|
|
.expect("transcribe");
|
|
assert!(
|
|
transcriber.detected_language().is_some(),
|
|
"auto mode should report a detected language after a decode"
|
|
);
|
|
}
|
|
}
|