From 177f1f7d90d3ed67f9bf64ee12432347c723d7dc Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Wed, 15 Jul 2026 19:35:16 -0500 Subject: [PATCH 01/10] feat(npu): catalog the merged Whisper decoder (decoder_model_merged.onnx) for KV-cache decode (T3.4) --- src-tauri/src/transcription/onnx_models.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src-tauri/src/transcription/onnx_models.rs b/src-tauri/src/transcription/onnx_models.rs index 083d935..a799b63 100644 --- a/src-tauri/src/transcription/onnx_models.rs +++ b/src-tauri/src/transcription/onnx_models.rs @@ -15,9 +15,16 @@ const REPO: &str = "onnx-community/whisper-base.en"; pub const DEFAULT_ONNX_MODEL: &str = "base.en"; /// The four artifacts the `OnnxTranscriber` needs, relative to the repo root. +/// The decoder is Optimum's *merged* export — both the no-past and with-past +/// branches in one graph, selected by a `use_cache_branch` input — which is +/// what enables the KV-cache decode loop (the plain `decoder_model.onnx` +/// re-ran the whole prefix every step). Existing installs migrate for free: +/// swapping this entry flips `is_installed` to false, the startup background +/// fetch re-runs `download`, and `download` skips the three files already on +/// disk, fetching only the merged decoder. const FILES: &[(&str, &str)] = &[ ("encoder_model.onnx", "onnx/encoder_model.onnx"), - ("decoder_model.onnx", "onnx/decoder_model.onnx"), + ("decoder_model_merged.onnx", "onnx/decoder_model_merged.onnx"), ("tokenizer.json", "tokenizer.json"), ("generation_config.json", "generation_config.json"), ]; @@ -45,7 +52,7 @@ pub fn model_files(id: &str) -> OnnxModelFiles { let dir = onnx_dir(id); OnnxModelFiles { encoder: dir.join("encoder_model.onnx"), - decoder: dir.join("decoder_model.onnx"), + decoder: dir.join("decoder_model_merged.onnx"), tokenizer: dir.join("tokenizer.json"), generation_config: dir.join("generation_config.json"), } @@ -125,6 +132,6 @@ mod tests { fn catalog_lists_the_four_required_artifacts() { assert_eq!(FILES.len(), 4); assert!(FILES.iter().any(|(n, _)| *n == "encoder_model.onnx")); - assert!(FILES.iter().any(|(n, _)| *n == "decoder_model.onnx")); + assert!(FILES.iter().any(|(n, _)| *n == "decoder_model_merged.onnx")); } } -- 2.34.1 From 402ffe45867e92f48c7df2612519140831cce2f4 Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Wed, 15 Jul 2026 19:48:14 -0500 Subject: [PATCH 02/10] feat(npu): KV-cache greedy decode + decoder on the OpenVINO GPU EP (T3.4) Merged-decoder (use_cache_branch/past_key_values) loop discovered from the graph at load; step 1 runs the no-cache branch over the prompt, later steps feed one token + cached KVs (decoder slots refreshed, encoder cross-attn KVs kept from step 1). Decoder session lands on the Intel iGPU via the OpenVINO GPU plugin (same staged runtime) with CPU-EP fallback and a WA_ONNX_DECODER_DEVICE=cpu|gpu override; DirectML backends keep the CPU decoder. Legacy decoder_model.onnx is removed once the merged graph loads. Measured (Core Ultra 5 135U, base.en, 19s TTS clip, identical transcripts): decoder on OpenVINO/GPU infer=1288ms vs CPU-EP infer=6098ms --- src-tauri/src/transcription/npu.rs | 249 ++++++++++++++++++++++++++--- 1 file changed, 226 insertions(+), 23 deletions(-) diff --git a/src-tauri/src/transcription/npu.rs b/src-tauri/src/transcription/npu.rs index b6fc256..6b0fbce 100644 --- a/src-tauri/src/transcription/npu.rs +++ b/src-tauri/src/transcription/npu.rs @@ -7,21 +7,24 @@ //! //! Split of work, validated by the T3.4 spike (encoder ~3.6× faster on NPU): //! - **encoder** (fixed `[1,80,3000]` shape) runs on the **accelerator** (NPU or -//! GPU) — the expensive graph, and the fixed shape accelerators want; -//! - **decoder** (dynamic, autoregressive) runs greedy on the **CPU** EP — a -//! dynamic KV loop is a poor accelerator fit and the cheap half anyway. -//! -//! ponytail: greedy, no-KV-cache decode (re-feeds the full token prefix each -//! step). Correct and simple; windows are short so the token count is small. -//! Add `decoder_with_past` only if the decoder shows up in a profile. +//! DirectML GPU) — the expensive graph, and the fixed shape accelerators want; +//! - **decoder** (dynamic, autoregressive) decodes greedily **with a KV cache** +//! (Optimum's merged export: `use_cache_branch` + `past_key_values`), so each +//! step feeds one token instead of re-running the whole prefix. On the NPU +//! path the decoder session itself goes to the **OpenVINO GPU EP** (Intel +//! iGPU, same runtime bundle) so sustained CPU stays near-idle; it falls back +//! to the CPU EP when no iGPU is usable, and `WA_ONNX_DECODER_DEVICE=cpu|gpu` +//! overrides the choice. DirectML backends keep the CPU decoder — the DML EP +//! handles per-step-growing KV shapes poorly. use super::{AudioWindow, SegmentSink, Transcriber, TrxError}; use crate::models::{BackendId, TranscriptSegment}; use ort::execution_providers::{ DirectMLExecutionProvider, ExecutionProviderDispatch, OpenVINOExecutionProvider, }; -use ort::session::Session; -use ort::value::Tensor; +use ort::session::{Session, SessionInputValue}; +use ort::value::{Tensor, ValueType}; +use std::borrow::Cow; use std::collections::HashMap; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; @@ -30,6 +33,36 @@ use std::sync::Mutex; /// Safety cap on generated tokens per 30 s window (Whisper's own max is 448). const MAX_NEW_TOKENS: usize = 224; +/// One KV-cache tensor pair of the merged decoder, discovered from the graph +/// at load: the `past_key_values.*` input, its matching `present.*` output, +/// whether it's decoder self-attention (refreshed every step) or encoder +/// cross-attention (computed once on the first step, then passed through +/// untouched by the cache branch), and the dummy shape to feed before any +/// past exists (see [`dummy_past_dims`]). +struct KvSlot { + past: String, + present: String, + self_attention: bool, + dummy_dims: Vec, +} + +/// `"past_key_values.3.decoder.key"` → `"present.3.decoder.key"` — Optimum's +/// fixed naming convention for the merged decoder's cache I/O. +fn present_name_for(past: &str) -> String { + format!("present.{}", past.trim_start_matches("past_key_values.")) +} + +/// The "no past yet" shape for a KV input: batch → 1, any other dynamic (−1) +/// dim → **1** (a dummy length-1 past — ORT's raw-data tensor creation rejects +/// zero-length dims, and the no-cache branch never reads the values anyway; +/// same dummy Optimum's own runner feeds), static dims kept. +fn dummy_past_dims(dims: &[i64]) -> Vec { + dims.iter() + .enumerate() + .map(|(i, &d)| if i == 0 || d < 0 { 1 } else { d as usize }) + .collect() +} + pub struct OnnxTranscriber { encoder: Mutex, decoder: Mutex, @@ -38,6 +71,13 @@ pub struct OnnxTranscriber { dec_ids: String, dec_hidden: String, dec_logits: String, + /// The merged decoder's KV-cache plumbing (see [`KvSlot`]) plus its + /// branch selector input. + kv_slots: Vec, + dec_use_cache: String, + /// Which EP the decoder session actually landed on ("OpenVINO/GPU" or + /// "CPU") — surfaced so `load_transcriber`'s engine log tells the truth. + decoder_ep: &'static str, detok: Detok, /// `[decoder_start_token_id, ...forced_decoder_ids]` — the fixed prompt /// prefix before generation begins. @@ -120,29 +160,76 @@ impl OnnxTranscriber { let hidden = Tensor::from_array((hidden_shape, hidden_data)) .map_err(|e| TrxError::Inference(e.to_string()))?; - // Greedy decode on the CPU. + // Greedy KV-cache decode. Step 1 runs the merged graph's no-cache + // branch over the full prompt and yields every present KV; later steps + // feed a single token plus the cache, refreshing only the decoder + // self-attention slots — the cache branch passes encoder + // cross-attention KVs through untouched, so the step-1 tensors stay + // authoritative (same contract as Optimum's own runner). + // ponytail: KVs round-trip host memory each step; wire ort IoBinding + // to pin them on-device if GPU-decoder profiling shows the copies matter. + let mut kv_cache: Vec> = self + .kv_slots + .iter() + .map(|slot| { + let len = slot.dummy_dims.iter().product::(); + Tensor::from_array((slot.dummy_dims.clone(), vec![0.0f32; len])) + }) + .collect::>() + .map_err(|e| TrxError::Inference(e.to_string()))?; + let mut tokens = self.initial_tokens.clone(); + let mut cached = false; // flips true once step 1 has filled kv_cache for _ in 0..MAX_NEW_TOKENS { - let ids: Vec = tokens.clone(); + let ids: Vec = if cached { + vec![*tokens.last().unwrap_or(&self.eot)] + } else { + tokens.clone() + }; let ids_tensor = Tensor::from_array(([1usize, ids.len()], ids)) .map_err(|e| TrxError::Inference(e.to_string()))?; + let branch = Tensor::from_array(([1usize], vec![cached])) + .map_err(|e| TrxError::Inference(e.to_string()))?; - let next = { + // Presents are extracted (copied) inside the outputs' borrow scope, + // then written back into kv_cache after it ends. + let (next, fresh_kvs) = { let mut dec = self .decoder .lock() .map_err(|_| TrxError::Inference("decoder mutex poisoned".into()))?; + let mut inputs: Vec<(Cow<'_, str>, SessionInputValue<'_>)> = vec![ + (Cow::from(self.dec_ids.as_str()), ids_tensor.view().into()), + (Cow::from(self.dec_hidden.as_str()), hidden.view().into()), + (Cow::from(self.dec_use_cache.as_str()), branch.view().into()), + ]; + for (slot, kv) in self.kv_slots.iter().zip(kv_cache.iter()) { + inputs.push((Cow::from(slot.past.as_str()), kv.view().into())); + } let outputs = dec - .run(ort::inputs![ - self.dec_ids.as_str() => ids_tensor.view(), - self.dec_hidden.as_str() => hidden.view(), - ]) + .run(inputs) .map_err(|e| TrxError::Inference(e.to_string()))?; let (shape, data) = outputs[self.dec_logits.as_str()] .try_extract_tensor::() .map_err(|e| TrxError::Inference(e.to_string()))?; - self.argmax_last(shape, data) + let next = self.argmax_last(shape, data); + + let mut fresh: Vec<(usize, Vec, Vec)> = Vec::new(); + for (i, slot) in self.kv_slots.iter().enumerate() { + if slot.self_attention || !cached { + let (s, d) = outputs[slot.present.as_str()] + .try_extract_tensor::() + .map_err(|e| TrxError::Inference(e.to_string()))?; + fresh.push((i, s.iter().map(|d| *d as usize).collect(), d.to_vec())); + } + } + (next, fresh) }; + for (i, dims, data) in fresh_kvs { + kv_cache[i] = Tensor::from_array((dims, data)) + .map_err(|e| TrxError::Inference(e.to_string()))?; + } + cached = true; if next == self.eot { break; @@ -180,6 +267,11 @@ impl OnnxTranscriber { best } + /// Which EP the decoder session landed on — "OpenVINO/GPU" or "CPU". + pub fn decoder_ep(&self) -> &'static str { + self.decoder_ep + } + fn segment(&self, text: String, start_ms: u64, end_ms: u64) -> Option { if text.is_empty() { return None; @@ -252,7 +344,7 @@ impl Transcriber for OnnxTranscriber { Self::ensure_runtime_env(runtime_dll); let encoder_path = model.join("encoder_model.onnx"); - let decoder_path = model.join("decoder_model.onnx"); + let decoder_path = model.join("decoder_model_merged.onnx"); let tokenizer_path = model.join("tokenizer.json"); let gen_config_path = model.join("generation_config.json"); @@ -263,11 +355,46 @@ impl Transcriber for OnnxTranscriber { .commit_from_file(&encoder_path) .map_err(|e| TrxError::Load(e.to_string()))?; - // Decoder on the default CPU EP (dynamic shapes; poor NPU fit). - let decoder = Session::builder() - .map_err(|e| TrxError::Load(e.to_string()))? - .commit_from_file(&decoder_path) - .map_err(|e| TrxError::Load(e.to_string()))?; + // Decoder session. The KV-cache loop is dynamic-shaped and + // autoregressive — a poor NPU fit — but it needn't burn CPU either: on + // the NPU path, try the same OpenVINO runtime's GPU plugin (Intel + // iGPU) first and fall back to the default CPU EP if no GPU is usable. + // `WA_ONNX_DECODER_DEVICE=cpu|gpu` overrides (debug escape hatch, like + // WA_NPU_RUNTIME_URL). DirectML backends keep the CPU decoder — the + // DML EP handles per-step-growing KV shapes poorly. + let want_gpu = match std::env::var("WA_ONNX_DECODER_DEVICE").as_deref() { + Ok("cpu") => false, + Ok("gpu") => true, + _ => matches!(backend, BackendId::Npu), + }; + let mut decoder_ep = "CPU"; + let mut decoder = None; + if want_gpu { + let gpu = Session::builder() + .and_then(|b| { + b.with_execution_providers([OpenVINOExecutionProvider::default() + .with_device_type("GPU") + .build() + .error_on_failure()]) + }) + .and_then(|b| b.commit_from_file(&decoder_path)); + match gpu { + Ok(s) => { + decoder_ep = "OpenVINO/GPU"; + decoder = Some(s); + } + Err(e) => tracing::warn!( + "OpenVINO GPU EP unavailable for the decoder ({e}); using the CPU EP" + ), + } + } + let decoder = match decoder { + Some(s) => s, + None => Session::builder() + .map_err(|e| TrxError::Load(e.to_string()))? + .commit_from_file(&decoder_path) + .map_err(|e| TrxError::Load(e.to_string()))?, + }; let enc_input = Self::find_input(&encoder, "input_features")?; let enc_output = Self::find_output(&encoder, "last_hidden_state") @@ -275,6 +402,53 @@ impl Transcriber for OnnxTranscriber { let dec_ids = Self::find_input(&decoder, "input_ids")?; let dec_hidden = Self::find_input(&decoder, "encoder_hidden")?; let dec_logits = Self::find_output(&decoder, "logits")?; + let dec_use_cache = Self::find_input(&decoder, "use_cache_branch")?; + + // KV-cache plumbing, discovered rather than hardcoded so the layer + // count comes from the graph (whisper-base: 6 layers → 24 slots). + let mut kv_slots: Vec = Vec::new(); + for input in &decoder.inputs { + if !input.name.starts_with("past_key_values.") { + continue; + } + let dims: Vec = match &input.input_type { + ValueType::Tensor { shape, .. } => shape.to_vec(), + other => { + return Err(TrxError::Load(format!( + "KV input '{}' has non-tensor type {other:?}", + input.name + ))) + } + }; + let present = present_name_for(&input.name); + if !decoder.outputs.iter().any(|o| o.name == present) { + return Err(TrxError::Load(format!( + "decoder missing present output '{present}'" + ))); + } + kv_slots.push(KvSlot { + self_attention: input.name.contains(".decoder."), + dummy_dims: dummy_past_dims(&dims), + past: input.name.clone(), + present, + }); + } + if kv_slots.is_empty() { + return Err(TrxError::Load( + "decoder has no past_key_values inputs — expected the merged (KV-cache) export" + .into(), + )); + } + + // A pre-KV-cache install leaves the old plain decoder behind — 208 MB + // of dead weight once the merged graph loads. Best-effort removal. + let legacy = model.join("decoder_model.onnx"); + if legacy.exists() { + match std::fs::remove_file(&legacy) { + Ok(()) => tracing::info!("removed superseded decoder_model.onnx"), + Err(e) => tracing::warn!("couldn't remove superseded decoder_model.onnx: {e}"), + } + } let detok = Detok::from_tokenizer_json(&tokenizer_path)?; let gen = GenConfig::from_file(&gen_config_path)?; @@ -290,6 +464,9 @@ impl Transcriber for OnnxTranscriber { dec_ids, dec_hidden, dec_logits, + kv_slots, + dec_use_cache, + decoder_ep, detok, initial_tokens, special_floor: gen.decoder_start, @@ -457,6 +634,28 @@ mod tests { assert_eq!(d.get(&'A'), Some(&b'A')); } + #[test] + fn present_name_follows_optimum_convention() { + assert_eq!( + present_name_for("past_key_values.3.decoder.key"), + "present.3.decoder.key" + ); + assert_eq!( + present_name_for("past_key_values.0.encoder.value"), + "present.0.encoder.value" + ); + } + + #[test] + fn dummy_past_dims_pins_dynamic_axes_to_one_and_keeps_static_ones() { + // whisper-base decoder self-attention past: [batch, 8, past_seq, 64] + // with batch/past_seq dynamic — both pin to 1 (ORT rejects 0-length + // dims for raw-data tensors; the no-cache branch ignores the values). + assert_eq!(dummy_past_dims(&[-1, 8, -1, 64]), vec![1, 8, 1, 64]); + // A static trailing shape is preserved (batch still pins to 1). + assert_eq!(dummy_past_dims(&[-1, 8, 1500, 64]), vec![1, 8, 1500, 64]); + } + #[test] fn byte_decoder_covers_all_256_bytes() { let d = byte_decoder(); @@ -481,10 +680,14 @@ mod tests { return; }; let wav = std::env::var("WA_NPU_TEST_WAV").expect("set WA_NPU_TEST_WAV"); + // Surface the engine's own EP-selection logs (GPU-decoder fallback + // warnings etc.) — the test harness has no subscriber otherwise. + let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); let t0 = std::time::Instant::now(); let t = OnnxTranscriber::load(Path::new(&model_dir), BackendId::Npu, None) .expect("load NPU transcriber"); let load_ms = t0.elapsed().as_millis(); + eprintln!("[spike] decoder EP: {}", t.decoder_ep()); let t1 = std::time::Instant::now(); let segs = t.transcribe_file(Path::new(&wav)).expect("transcribe"); let infer_ms = t1.elapsed().as_millis(); -- 2.34.1 From 7f7de48ecc5a5aba48969471cb2fea5aac628466 Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Wed, 15 Jul 2026 19:48:15 -0500 Subject: [PATCH 03/10] feat(npu): engine log names the decoder EP alongside the encoder EP --- src-tauri/src/commands.rs | 130 ++++++++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 47 deletions(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 11867ee..e7dc258 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -194,17 +194,17 @@ fn load_transcriber( match OnnxTranscriber::load(&dir, backend, language) { Ok(t) => { // The definitive "is the accelerator actually engaged" line - // (visible with `npm run tauri dev`): the encoder runs on - // the EP below; the autoregressive decoder always runs on - // the CPU EP, which is why Task Manager shows only short - // periodic NPU spikes alongside sustained CPU load. + // (visible with `npm run tauri dev`): encoder EP from the + // accel path, decoder EP from wherever its session landed + // (OpenVINO/GPU on NPU systems, CPU otherwise/on fallback). tracing::info!( - "transcription engine: ONNX {} (encoder EP: {}, decoder: CPU)", + "transcription engine: ONNX {} (encoder EP: {}, decoder EP: {})", onnx_models::DEFAULT_ONNX_MODEL, match path { AccelPath::OnnxOpenVino => "OpenVINO/NPU", _ => "DirectML/GPU", - } + }, + t.decoder_ep() ); return Ok((Box::new(t), backend)); } @@ -347,7 +347,9 @@ async fn attribute_split( spans.sort_by_key(|s| s.start_ms); // Uniform naming: "You" -> "You", far speakers -> "Speaker 2", "Speaker 3"…. let labels = crate::diarization::voiceprint::first_appearance_order(&spans); - Some(crate::diarization::voiceprint::build_name_map(&labels, "You")) + Some(crate::diarization::voiceprint::build_name_map( + &labels, "You", + )) } /// The distinct speakers seen in `segments` so far, in first-appearance order, @@ -655,28 +657,23 @@ pub async fn start_recording( .lock() .unwrap_or_else(|e| e.into_inner()) .clone(); - if attribute_split( - diarizer.clone(), - wav_path_for_diar.clone(), - &mut snapshot, - ) - .await - .map(|auto_names| { - let mut names = names_for_diar.lock().unwrap_or_else(|e| e.into_inner()); - // Never overwrite a name already set (user rename or a - // prior pass) — same guard as the post-stop pass. - for (label, name) in auto_names { - names.entry(label).or_insert(name); - } - }) - .is_none() + if attribute_split(diarizer.clone(), wav_path_for_diar.clone(), &mut snapshot) + .await + .map(|auto_names| { + let mut names = + names_for_diar.lock().unwrap_or_else(|e| e.into_inner()); + // Never overwrite a name already set (user rename or a + // prior pass) — same guard as the post-stop pass. + for (label, name) in auto_names { + names.entry(label).or_insert(name); + } + }) + .is_none() { continue; // pass failed (already logged); retry next tick } - let relabeled: HashMap = snapshot - .into_iter() - .map(|s| (s.id, s.speaker)) - .collect(); + let relabeled: HashMap = + snapshot.into_iter().map(|s| (s.id, s.speaker)).collect(); let mut segs = segments_for_diar.lock().unwrap_or_else(|e| e.into_inner()); // Snapshot prior labels so only segments whose speaker // actually changed this pass get re-emitted (ids are stable; @@ -891,7 +888,8 @@ pub async fn stop_recording( if already_named { continue; } - if let Err(e) = state.store.rename_speaker(&meeting_id, &label, &name).await { + if let Err(e) = state.store.rename_speaker(&meeting_id, &label, &name).await + { tracing::warn!("failed to persist auto speaker name: {e}"); continue; } @@ -1757,8 +1755,11 @@ fn pick_realtime_recommendation( .max_by(|a, b| { let sa = sizes.get(&a.model).copied().unwrap_or(0); let sb = sizes.get(&b.model).copied().unwrap_or(0); - sa.cmp(&sb) - .then(b.rtf.partial_cmp(&a.rtf).unwrap_or(std::cmp::Ordering::Equal)) + sa.cmp(&sb).then( + b.rtf + .partial_cmp(&a.rtf) + .unwrap_or(std::cmp::Ordering::Equal), + ) }) .map(|r| StressRecommendation { backend: r.backend.clone(), @@ -2525,10 +2526,17 @@ async fn run_import_pipeline( }) .await; match transcode { - Ok(Ok(())) => { - emit_import_progress(&app, &meeting_id, "prepare", "done", Some(t.elapsed().as_millis()), None) + Ok(Ok(())) => emit_import_progress( + &app, + &meeting_id, + "prepare", + "done", + Some(t.elapsed().as_millis()), + None, + ), + Ok(Err(e)) => { + return fail_import(&app, &store, &meeting_id, "prepare", &e.to_string()).await } - Ok(Err(e)) => return fail_import(&app, &store, &meeting_id, "prepare", &e.to_string()).await, Err(e) => return fail_import(&app, &store, &meeting_id, "prepare", &e.to_string()).await, } @@ -2552,11 +2560,22 @@ async fn run_import_pipeline( .await; let (mut segments, resolved_language) = match transcribed { Ok(Ok(v)) => { - emit_import_progress(&app, &meeting_id, "transcribe", "done", Some(t.elapsed().as_millis()), None); + emit_import_progress( + &app, + &meeting_id, + "transcribe", + "done", + Some(t.elapsed().as_millis()), + None, + ); v } - Ok(Err(e)) => return fail_import(&app, &store, &meeting_id, "transcribe", &e.to_string()).await, - Err(e) => return fail_import(&app, &store, &meeting_id, "transcribe", &e.to_string()).await, + Ok(Err(e)) => { + return fail_import(&app, &store, &meeting_id, "transcribe", &e.to_string()).await + } + Err(e) => { + return fail_import(&app, &store, &meeting_id, "transcribe", &e.to_string()).await + } }; // Phase 3: diarize — one pass if the models are installed, else every line @@ -2578,7 +2597,14 @@ async fn run_import_pipeline( Err(e) => tracing::warn!("import diarization task failed: {e}"), } } - emit_import_progress(&app, &meeting_id, "diarize", "done", Some(t.elapsed().as_millis()), None); + emit_import_progress( + &app, + &meeting_id, + "diarize", + "done", + Some(t.elapsed().as_millis()), + None, + ); // Phase 4: finalize — persist segments, notes, seal audio. emit_import_progress(&app, &meeting_id, "finalize", "active", None, None); @@ -2626,7 +2652,14 @@ async fn run_import_pipeline( } } } - emit_import_progress(&app, &meeting_id, "finalize", "done", Some(t.elapsed().as_millis()), None); + emit_import_progress( + &app, + &meeting_id, + "finalize", + "done", + Some(t.elapsed().as_millis()), + None, + ); let _ = app.emit( "transcript://finalized", @@ -2643,7 +2676,9 @@ async fn fail_import( phase: &str, message: &str, ) { - let _ = store.set_meeting_status(&meeting_id.to_string(), "error").await; + let _ = store + .set_meeting_status(&meeting_id.to_string(), "error") + .await; emit_import_progress(app, meeting_id, phase, "error", None, Some(message)); } @@ -2926,8 +2961,8 @@ fn fold_wav(wav: &[u8], out_channels: u16) -> Option> { (hound::SampleFormat::Int, 16) => { let s: Vec = reader.samples::().map_while(Result::ok).collect(); for frame in s.chunks(ch) { - let avg = (frame.iter().map(|&x| x as i32).sum::() - / frame.len() as i32) as i16; + let avg = + (frame.iter().map(|&x| x as i32).sum::() / frame.len() as i32) as i16; for _ in 0..out_channels { writer.write_sample(avg).ok()?; } @@ -3125,10 +3160,7 @@ pub async fn bulk_export_meetings( /// so re-importing never collides with existing meetings. Returns the count /// imported. #[tauri::command] -pub async fn import_meeting_bundle( - state: State<'_, AppState>, - dir: String, -) -> WaResult { +pub async fn import_meeting_bundle(state: State<'_, AppState>, dir: String) -> WaResult { let root = PathBuf::from(&dir); let mut bundles: Vec = Vec::new(); if root.join("meeting.json").exists() { @@ -3977,7 +4009,11 @@ only, with no preamble or commentary."; let notes = notes.trim(); let user = format!( "My rough notes:\n{}\n\nTranscript:\n{}", - if notes.is_empty() { "(none yet)" } else { notes }, + if notes.is_empty() { + "(none yet)" + } else { + notes + }, prompt.transcript ); provider @@ -4895,8 +4931,8 @@ pub(crate) fn spawn_auto_sync( let app = app.clone(); tauri::async_runtime::spawn(async move { match enqueue_meeting_sync(store.as_ref(), &meeting_id, None, true).await { - Ok(0) => {} // nothing changed - Ok(_) => pump_sync(&app, store.as_ref()).await, // upload the changes + Ok(0) => {} // nothing changed + Ok(_) => pump_sync(&app, store.as_ref()).await, // upload the changes Err(e) => tracing::warn!("auto-sync enqueue failed: {e:?}"), } }); -- 2.34.1 From 4a3f8de67943cce00930658730e5759a25e3253d Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Wed, 15 Jul 2026 19:48:16 -0500 Subject: [PATCH 04/10] =?UTF-8?q?docs(adr):=200004=20update=20=E2=80=94=20?= =?UTF-8?q?NPU=20path=20is=20OpenVINO;=20decoder=20split=20to=20iGPU=20wit?= =?UTF-8?q?h=20KV=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/adr/0004-hardware-acceleration.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/adr/0004-hardware-acceleration.md b/docs/adr/0004-hardware-acceleration.md index ac0b6c4..fba6d4f 100644 --- a/docs/adr/0004-hardware-acceleration.md +++ b/docs/adr/0004-hardware-acceleration.md @@ -44,3 +44,20 @@ Detection mapping: ## Revisit if Windows ML supersedes DirectML for our models, or a vendor SDK gives materially better NPU throughput than the ONNX/DirectML path. + +## Update (2026-07-16): NPU path uses OpenVINO; decoder split off the CPU + +As built (T3.4 + follow-up), the NPU tier runs on **ONNX Runtime + the OpenVINO EP** +(`device_type=NPU`), not DirectML — Intel's prebuilt OpenVINO ORT runtime is downloaded on +demand (DirectML remains the *GPU* fallback EP for AMD/Intel in non-Vulkan builds). The engine +splits Whisper across devices: + +- **encoder** (fixed shape): OpenVINO **NPU**; +- **decoder** (autoregressive, KV-cache greedy via Optimum's merged export): OpenVINO **GPU** + (Intel iGPU, same runtime bundle) when available, else the CPU EP — + `WA_ONNX_DECODER_DEVICE=cpu|gpu` overrides. + +Rationale: the dynamic-shaped decode loop is a poor NPU fit, and pre-KV-cache it re-ran the full +token prefix every step on the CPU — the dominant CPU cost while transcribing. With the merged +decoder + iGPU placement, sustained CPU during transcription is near-idle on NPU+iGPU systems. +The engine logs both EPs at load (`transcription engine: ONNX …`). -- 2.34.1 From 89bd7c3b7edd3f880bf9aee0e9f4ede2592c408e Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Wed, 15 Jul 2026 20:14:30 -0500 Subject: [PATCH 05/10] chore: bump version 0.7.3 --- package.json | 2 +- src-tauri/Cargo.lock | 2 +- src-tauri/Cargo.toml | 2 +- src-tauri/tauri.conf.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 5de12bc..1f12387 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "whispassist", "private": true, - "version": "0.7.2", + "version": "0.7.3", "type": "module", "description": "Privacy-first, fully local Windows meeting assistant.", "license": "MIT OR Apache-2.0", diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index b036b3d..c4dd311 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -6088,7 +6088,7 @@ dependencies = [ [[package]] name = "whispassist" -version = "0.7.2" +version = "0.7.3" dependencies = [ "argon2", "async-trait", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 3db30ed..d13135d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "whispassist" -version = "0.7.2" +version = "0.7.3" description = "Privacy-first, fully local Windows meeting assistant" authors = ["WhispAssist contributors"] license = "MIT OR Apache-2.0" diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 98af576..09b6312 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "WhispAssist", - "version": "0.7.2", + "version": "0.7.3", "identifier": "bet.dou.whispassist", "build": { "frontendDist": "../dist", -- 2.34.1 From 29bb636cb54bb164684d5cc6fc7718f0df8ef681 Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Wed, 15 Jul 2026 20:28:18 -0500 Subject: [PATCH 06/10] feat(i18n): Settings calendar 'requires readpst' string --- src/lib/i18n/en.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/i18n/en.json b/src/lib/i18n/en.json index f9a4bb0..47db52d 100644 --- a/src/lib/i18n/en.json +++ b/src/lib/i18n/en.json @@ -107,6 +107,7 @@ "settings.calendar.title": "Calendar & Outlook .pst", "settings.calendar.intro_1": "Import events and attendees from a local Outlook", "settings.calendar.intro_2": "backup — read-only, nothing is written back to the file. Nothing leaves this device.", + "settings.calendar.requires": "Needs this on your PATH (not bundled):", "settings.calendar.pst_file": ".pst file", "settings.calendar.no_file": "No file selected", "settings.calendar.browse": "Browse…", -- 2.34.1 From c9643cd79b2993208379dccd665beaae7adf13eb Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Wed, 15 Jul 2026 20:28:19 -0500 Subject: [PATCH 07/10] feat(calendar): libpst (readpst) download button in Settings, matching the Import dialog's tool buttons (FR-CAL-1, ADR-0008) --- src/lib/views/Settings.svelte | 42 ++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/lib/views/Settings.svelte b/src/lib/views/Settings.svelte index 360716b..1c916f4 100644 --- a/src/lib/views/Settings.svelte +++ b/src/lib/views/Settings.svelte @@ -286,6 +286,11 @@ } // ---- Calendar / .pst import (T6.1/T6.2/T6.3, FR-CAL-1/2) ---- + // .pst import shells out to readpst (libpst, ADR-0008) — external like + // ffmpeg/yt-dlp, so link the same kind of download button as the Import + // dialog's. ezwinports is the maintained Windows build of libpst. + const LIBPST_URL = + "https://sourceforge.net/projects/ezwinports/files/libpst-0.6.63-w32-bin.zip/download"; let pstPath = $state(""); let pstPassword = $state(""); let eventTitleFilter = $state(""); @@ -987,7 +992,9 @@

{t("settings.hardware.stress_hint")}

{#if stressProgress}

{t("settings.hardware.stress_progress", { pair: stressProgress })}

@@ -1344,6 +1351,12 @@ {t("settings.calendar.intro_1")} .pst {t("settings.calendar.intro_2")}

+
+ {t("settings.calendar.requires")} + +