From f8731adae379f65c8cdf45d0bb2377c4b7321622 Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Thu, 2 Jul 2026 16:14:15 -0500 Subject: [PATCH] =?UTF-8?q?feat(transcription):=20OnnxNpuTranscriber=20?= =?UTF-8?q?=E2=80=94=20NPU=20encoder=20+=20CPU=20greedy=20decode=20(T3.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src-tauri/src/transcription/npu.rs | 454 +++++++++++++++++++++++++++++ 1 file changed, 454 insertions(+) create mode 100644 src-tauri/src/transcription/npu.rs diff --git a/src-tauri/src/transcription/npu.rs b/src-tauri/src/transcription/npu.rs new file mode 100644 index 0000000..e8521ce --- /dev/null +++ b/src-tauri/src/transcription/npu.rs @@ -0,0 +1,454 @@ +//! NPU transcriber: Whisper ONNX on the Intel NPU via ONNX Runtime + OpenVINO +//! (T3.4, ADR-0004). Same `Transcriber` trait and `TranscriptSegment` output as +//! the whisper.cpp path, so callers never branch on engine. +//! +//! Split of work, validated by the T3.4 spike (encoder ~3.6× faster on NPU): +//! - **encoder** (fixed `[1,80,3000]` shape) runs on the **NPU** via OpenVINO — +//! the expensive graph, and the shape NPUs want; +//! - **decoder** (dynamic, autoregressive) runs greedy on the **CPU** EP — a +//! dynamic KV loop is a poor NPU fit and the cheap half anyway. +//! +//! ponytail: greedy, no-KV-cache decode (re-feeds the full token prefix each +//! step). Correct and simple; windows are short so the token count is small. +//! Add `decoder_with_past` only if the decoder shows up in a profile. + +use super::{AudioWindow, SegmentSink, Transcriber, TrxError}; +use crate::models::{BackendId, TranscriptSegment}; +use ort::execution_providers::OpenVINOExecutionProvider; +use ort::session::Session; +use ort::value::Tensor; +use std::collections::HashMap; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; + +/// Safety cap on generated tokens per 30 s window (Whisper's own max is 448). +const MAX_NEW_TOKENS: usize = 224; + +pub struct OnnxNpuTranscriber { + encoder: Mutex, + decoder: Mutex, + enc_input: String, + enc_output: String, + dec_ids: String, + dec_hidden: String, + dec_logits: String, + detok: Detok, + /// `[decoder_start_token_id, ...forced_decoder_ids]` — the fixed prompt + /// prefix before generation begins. + initial_tokens: Vec, + /// Special tokens start here (`decoder_start_token_id`); anything ≥ this is + /// masked out of argmax so only text tokens (or `eot`) can be emitted. + special_floor: i64, + eot: i64, + next_id: AtomicU64, +} + +impl OnnxNpuTranscriber { + /// Points `ort` (load-dynamic) at the on-demand-downloaded runtime and puts + /// its DLLs on the search path, unless the caller already set `ORT_DYLIB_PATH` + /// (the test harness does, to target a dev runtime). Idempotent. + fn ensure_runtime_env() { + if std::env::var_os("ORT_DYLIB_PATH").is_some() { + return; + } + let dll = crate::paths::npu_runtime_dll(); + std::env::set_var("ORT_DYLIB_PATH", &dll); + if let Some(dir) = dll.parent() { + let path = std::env::var_os("PATH").unwrap_or_default(); + let mut dirs = vec![dir.to_path_buf()]; + dirs.extend(std::env::split_paths(&path)); + if let Ok(joined) = std::env::join_paths(dirs) { + std::env::set_var("PATH", joined); + } + } + } + + fn find_input(session: &Session, needle: &str) -> Result { + session + .inputs + .iter() + .find(|i| i.name.contains(needle)) + .map(|i| i.name.to_string()) + .ok_or_else(|| TrxError::Load(format!("model missing input '{needle}'"))) + } + + fn find_output(session: &Session, needle: &str) -> Result { + session + .outputs + .iter() + .find(|o| o.name.contains(needle)) + .map(|o| o.name.to_string()) + .ok_or_else(|| TrxError::Load(format!("model missing output '{needle}'"))) + } + + /// Full encode→decode over one window's samples, returning the trimmed text. + fn infer(&self, samples: &[f32]) -> Result { + let mel = super::mel::log_mel_spectrogram(samples); + let enc_in = Tensor::from_array(([1usize, super::mel::N_MELS, super::mel::N_FRAMES], mel)) + .map_err(|e| TrxError::Inference(e.to_string()))?; + + // Encoder on the NPU. Copy hidden states out while the session is locked + // (outputs borrow the session). + let (hidden_shape, hidden_data) = { + let mut enc = self + .encoder + .lock() + .map_err(|_| TrxError::Inference("encoder mutex poisoned".into()))?; + let outputs = enc + .run(ort::inputs![self.enc_input.as_str() => enc_in.view()]) + .map_err(|e| TrxError::Inference(e.to_string()))?; + let (shape, data) = outputs[self.enc_output.as_str()] + .try_extract_tensor::() + .map_err(|e| TrxError::Inference(e.to_string()))?; + ( + shape.iter().map(|d| *d as usize).collect::>(), + data.to_vec(), + ) + }; + let hidden = Tensor::from_array((hidden_shape, hidden_data)) + .map_err(|e| TrxError::Inference(e.to_string()))?; + + // Greedy decode on the CPU. + let mut tokens = self.initial_tokens.clone(); + for _ in 0..MAX_NEW_TOKENS { + let ids: Vec = tokens.clone(); + let ids_tensor = Tensor::from_array(([1usize, ids.len()], ids)) + .map_err(|e| TrxError::Inference(e.to_string()))?; + + let next = { + let mut dec = self + .decoder + .lock() + .map_err(|_| TrxError::Inference("decoder mutex poisoned".into()))?; + let outputs = dec + .run(ort::inputs![ + self.dec_ids.as_str() => ids_tensor.view(), + self.dec_hidden.as_str() => hidden.view(), + ]) + .map_err(|e| TrxError::Inference(e.to_string()))?; + let (shape, data) = outputs[self.dec_logits.as_str()] + .try_extract_tensor::() + .map_err(|e| TrxError::Inference(e.to_string()))?; + self.argmax_last(shape, data) + }; + + if next == self.eot { + break; + } + tokens.push(next); + } + + Ok(self + .detok + .decode(&tokens[self.initial_tokens.len()..]) + .trim() + .to_string()) + } + + /// Argmax over the final position's logits, masking special tokens so only + /// text (or `eot`) can win — greedy, no timestamps. + fn argmax_last(&self, shape: &[i64], data: &[f32]) -> i64 { + let vocab = shape[shape.len() - 1] as usize; + let seq = shape[shape.len() - 2] as usize; + let row = &data[(seq - 1) * vocab..seq * vocab]; + let mut best = self.eot; + let mut best_val = f32::MIN; + for (id, &v) in row.iter().enumerate() { + let id = id as i64; + // Mask specials, but never mask eot so decode can always terminate + // regardless of how the config orders eot vs. the special floor. + if id >= self.special_floor && id != self.eot { + continue; + } + if v > best_val { + best_val = v; + best = id; + } + } + best + } + + fn segment(&self, text: String, start_ms: u64, end_ms: u64) -> Option { + if text.is_empty() { + return None; + } + Some(TranscriptSegment { + id: self.next_id.fetch_add(1, Ordering::SeqCst), + start_ms, + end_ms: end_ms.max(start_ms), + speaker: "S1".to_string(), // diarization lands in Phase 4 + text, + confidence: None, // greedy decode exposes no per-segment score yet + interim: false, + }) + } +} + +impl Transcriber for OnnxNpuTranscriber { + /// `model` is the directory holding the ONNX artifacts (see `onnx_models`). + /// `backend` is expected to be `Npu`; a non-NPU value is rejected so the + /// dispatcher can fall back to whisper.cpp rather than us guessing. + fn load(model: &Path, backend: BackendId) -> Result { + if backend != BackendId::Npu { + return Err(TrxError::Load(format!( + "OnnxNpuTranscriber only serves the NPU backend, got {backend:?}" + ))); + } + Self::ensure_runtime_env(); + + let encoder_path = model.join("encoder_model.onnx"); + let decoder_path = model.join("decoder_model.onnx"); + let tokenizer_path = model.join("tokenizer.json"); + let gen_config_path = model.join("generation_config.json"); + + // Encoder pinned to the NPU. error_on_failure makes a failed NPU + // registration LOUD (Err) instead of a silent CPU fallback, so the + // dispatcher can cleanly drop to the whisper.cpp path (T3.4 spike lesson). + let encoder = Session::builder() + .map_err(|e| TrxError::Load(e.to_string()))? + .with_execution_providers([OpenVINOExecutionProvider::default() + .with_device_type("NPU") + .build() + .error_on_failure()]) + .map_err(|e| TrxError::Load(e.to_string()))? + .commit_from_file(&encoder_path) + .map_err(|e| TrxError::Load(e.to_string()))?; + + // Decoder on the default CPU EP (dynamic shapes; poor NPU fit). + let decoder = Session::builder() + .map_err(|e| TrxError::Load(e.to_string()))? + .commit_from_file(&decoder_path) + .map_err(|e| TrxError::Load(e.to_string()))?; + + let enc_input = Self::find_input(&encoder, "input_features")?; + let enc_output = Self::find_output(&encoder, "last_hidden_state") + .or_else(|_| Self::find_output(&encoder, "hidden"))?; + let dec_ids = Self::find_input(&decoder, "input_ids")?; + let dec_hidden = Self::find_input(&decoder, "encoder_hidden")?; + let dec_logits = Self::find_output(&decoder, "logits")?; + + let detok = Detok::from_tokenizer_json(&tokenizer_path)?; + let gen = GenConfig::from_file(&gen_config_path)?; + + let mut initial_tokens = vec![gen.decoder_start]; + initial_tokens.extend(gen.forced); + + Ok(Self { + encoder: Mutex::new(encoder), + decoder: Mutex::new(decoder), + enc_input, + enc_output, + dec_ids, + dec_hidden, + dec_logits, + detok, + initial_tokens, + special_floor: gen.decoder_start, + eot: gen.eot, + next_id: AtomicU64::new(0), + }) + } + + fn transcribe_stream(&self, audio: AudioWindow, out: SegmentSink) -> Result<(), TrxError> { + let text = self.infer(&audio.samples)?; + let dur_ms = (audio.samples.len() as u64 * 1000) / super::mel::SAMPLE_RATE as u64; + if let Some(seg) = self.segment(text, audio.offset_ms, audio.offset_ms + dur_ms) { + let _ = out.send(seg); + } + Ok(()) + } + + fn transcribe_file(&self, wav: &Path) -> Result, TrxError> { + let samples = + crate::audio::read_wav_mono_16k(wav).map_err(|e| TrxError::Load(e.to_string()))?; + // The encoder is fixed at one 30 s window, so chunk long audio and offset + // each chunk's segment in time. + let chunk = super::mel::N_SAMPLES; + let mut out = Vec::new(); + for (i, part) in samples.chunks(chunk).enumerate() { + let text = self.infer(part)?; + let start_ms = (i * chunk) as u64 * 1000 / super::mel::SAMPLE_RATE as u64; + let dur_ms = (part.len() as u64 * 1000) / super::mel::SAMPLE_RATE as u64; + if let Some(seg) = self.segment(text, start_ms, start_ms + dur_ms) { + out.push(seg); + } + } + Ok(out) + } +} + +/// Whisper's generation config: the decoder-start token, end token, and any +/// forced prefix tokens (e.g. `<|notimestamps|>`). +struct GenConfig { + decoder_start: i64, + eot: i64, + forced: Vec, +} + +impl GenConfig { + fn from_file(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|e| TrxError::Load(format!("gen config: {e}")))?; + let v: serde_json::Value = + serde_json::from_str(&text).map_err(|e| TrxError::Load(format!("gen config: {e}")))?; + let decoder_start = v["decoder_start_token_id"] + .as_i64() + .ok_or_else(|| TrxError::Load("gen config missing decoder_start_token_id".into()))?; + // eos may be a scalar or an array; take the first if an array. + let eot = v["eos_token_id"] + .as_i64() + .or_else(|| { + v["eos_token_id"] + .as_array() + .and_then(|a| a.first()?.as_i64()) + }) + .unwrap_or(decoder_start - 1); + // forced_decoder_ids: [[pos, token], ...], applied in position order. + let mut forced_pairs: Vec<(i64, i64)> = v["forced_decoder_ids"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|p| { + let p = p.as_array()?; + Some((p.first()?.as_i64()?, p.get(1)?.as_i64()?)) + }) + .collect() + }) + .unwrap_or_default(); + forced_pairs.sort_by_key(|(pos, _)| *pos); + let forced = forced_pairs.into_iter().map(|(_, tok)| tok).collect(); + Ok(Self { + decoder_start, + eot, + forced, + }) + } +} + +/// Minimal Whisper byte-level BPE **detokenizer** (ids → text). We only decode, +/// so this avoids pulling the heavyweight `tokenizers` crate (and its C deps): +/// read the vocab from `tokenizer.json`, then invert GPT-2's byte↔unicode map. +struct Detok { + id_to_token: HashMap, + byte_decoder: HashMap, +} + +impl Detok { + fn from_tokenizer_json(path: &Path) -> Result { + let text = + std::fs::read_to_string(path).map_err(|e| TrxError::Load(format!("tokenizer: {e}")))?; + let v: serde_json::Value = + serde_json::from_str(&text).map_err(|e| TrxError::Load(format!("tokenizer: {e}")))?; + let vocab = v["model"]["vocab"] + .as_object() + .ok_or_else(|| TrxError::Load("tokenizer.json missing model.vocab".into()))?; + let id_to_token = vocab + .iter() + .filter_map(|(tok, id)| Some((id.as_i64()?, tok.clone()))) + .collect(); + Ok(Self { + id_to_token, + byte_decoder: byte_decoder(), + }) + } + + fn decode(&self, ids: &[i64]) -> String { + let mut bytes = Vec::new(); + for id in ids { + if let Some(tok) = self.id_to_token.get(id) { + for ch in tok.chars() { + if let Some(b) = self.byte_decoder.get(&ch) { + bytes.push(*b); + } + } + } + } + String::from_utf8_lossy(&bytes).into_owned() + } +} + +/// GPT-2 / Whisper `bytes_to_unicode`, inverted to map each printable proxy +/// char back to its raw byte. +fn byte_decoder() -> HashMap { + let mut bs: Vec = Vec::new(); + for b in b'!'..=b'~' { + bs.push(b as u32); + } + for b in 0xA1u32..=0xAC { + bs.push(b); + } + for b in 0xAEu32..=0xFF { + bs.push(b); + } + let mut cs = bs.clone(); + let mut n = 0u32; + for b in 0u32..256 { + if !bs.contains(&b) { + bs.push(b); + cs.push(256 + n); + n += 1; + } + } + bs.iter() + .zip(cs.iter()) + .filter_map(|(&b, &c)| Some((char::from_u32(c)?, b as u8))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn byte_decoder_maps_gpt2_space_proxy() { + // 'Ġ' (U+0120 = 256+32) is GPT-2's proxy for a space byte (0x20). + let d = byte_decoder(); + assert_eq!(d.get(&'Ġ'), Some(&b' ')); + // Printable ASCII maps to itself. + assert_eq!(d.get(&'A'), Some(&b'A')); + } + + #[test] + fn byte_decoder_covers_all_256_bytes() { + let d = byte_decoder(); + let mut seen = [false; 256]; + for &b in d.values() { + seen[b as usize] = true; + } + assert!( + seen.iter().all(|&s| s), + "byte decoder must be a bijection over 0..256" + ); + } + + /// Real NPU inference, opt-in (needs the runtime + a downloaded model + a + /// WAV, none of which exist in CI). Run locally after the T3.4 spike setup: + /// ORT_DYLIB_PATH, PATH, WA_NPU_MODEL_DIR, WA_NPU_TEST_WAV[, WA_NPU_EXPECT] + #[test] + #[ignore = "requires NPU runtime + model + wav; run manually"] + fn npu_transcribes_speech() { + let Some(model_dir) = std::env::var_os("WA_NPU_MODEL_DIR") else { + eprintln!("skip: set WA_NPU_MODEL_DIR"); + return; + }; + let wav = std::env::var("WA_NPU_TEST_WAV").expect("set WA_NPU_TEST_WAV"); + let t = OnnxNpuTranscriber::load(Path::new(&model_dir), BackendId::Npu) + .expect("load NPU transcriber"); + let segs = t.transcribe_file(Path::new(&wav)).expect("transcribe"); + let text = segs + .iter() + .map(|s| s.text.as_str()) + .collect::>() + .join(" ") + .to_lowercase(); + eprintln!("NPU transcript: {text:?}"); + assert!(!text.trim().is_empty(), "transcript was empty"); + if let Ok(expect) = std::env::var("WA_NPU_EXPECT") { + assert!( + text.contains(&expect.to_lowercase()), + "transcript {text:?} missing expected {expect:?}" + ); + } + } +}