diff --git a/README.md b/README.md index 5ed3c8a..5ef4745 100644 --- a/README.md +++ b/README.md @@ -11,36 +11,39 @@ on-device acceleration (**NPU → GPU → CPU**), labels speakers, structures th Markdown notes, and optionally augments them with a locally hosted LLM (Ollama). Audio and transcripts **never leave the machine** unless you explicitly configure a destination. -> **Status: working application (v0.5.2).** Capture (system audio **+ your microphone**, with a -> live **dual level meter**), transcription (CPU / Intel NPU / Vulkan GPU) with a **fluid live -> transcript**, speaker diarization, storage + crash recovery, local-LLM summaries, AI tags, +> **Status: working application (v0.7.3).** Capture (system audio **+ your microphone**, with a +> live **dual level meter** and an **M-key mic mute**), transcription (CPU / Intel NPU / Vulkan +> GPU) with a **fluid live transcript**, speaker diarization that labels **You** live from your +> own mic channel, storage + crash recovery, local-LLM summaries, AI tags, **AI-enhanced notes**, > opt-in recording with **in-app playback**, at-rest encryption, self-hosted sync, **Outlook -> `.pst`/calendar import with optional auto-record**, **importing an existing recording from a -> file or URL**, and a loopback **MCP server** for coding-agent handoff are all implemented and -> ship as a single signed **MSI + NSIS** universal installer. Build order and remaining tasks -> are in [`docs/05-roadmap.md`](docs/05-roadmap.md). +> `.pst`/calendar import with optional auto-record**, **background import of an existing +> recording from a file or URL** (with a step-by-step progress tracker), **close-to-tray**, and a +> loopback **MCP server** for coding-agent handoff are all implemented and ship as a single +> **MSI + NSIS** universal installer. On Intel Core Ultra machines, transcription runs with the +> **CPU near-idle**: Whisper's encoder on the **NPU**, its decoder on the **iGPU**. Build order +> and remaining tasks are in [`docs/05-roadmap.md`](docs/05-roadmap.md). ## Why WhispAssist — Granola vs Meetily vs WhispAssist -| | **Granola** | **Meetily** | **WhispAssist** | -|---|---|---|---| -| **Positioning** | Cloud AI notepad | Open-source self-hosted assistant | Local, Windows-native assistant | -| **Platform** | macOS, Windows | macOS, Windows, Linux | Windows 10/11 | -| **Bot-free system-audio capture** | ✅ | ✅ | ✅ WASAPI loopback | -| **Transcription** | ☁️ cloud | ✅ local (Whisper) | ✅ local (whisper.cpp) | -| **NPU / GPU auto-acceleration** | n/a (cloud) | ⚠️ CPU/GPU, manual | ✅ **NPU→NVIDIA→AMD→Intel→CPU** ladder, one binary | -| **Speaker diarization** | ✅ cloud | ⚠️ limited | ✅ offline (sherpa-onnx) | -| **Summaries / notes AI** | ☁️ cloud LLM | ✅ local (Ollama) / BYO | ✅ local (Ollama, localhost **or LAN**) + optional hosted | -| **Interface localization (UI language)** | ⚠️ English-only UI (multilingual transcription) | ⚠️ English-only UI (more planned) | ✅ **20 languages** + RTL (Arabic/Urdu) | -| **Default data egress** | ☁️ audio + notes to cloud | 🔒 local (cloud optional) | 🔒 **none** — everything off by default, allowlist-enforced | -| **Calendar / Outlook `.pst` context** | ✅ cloud calendar | ❌ | ✅ local `.pst` import + **auto-record on events** | -| **Import an existing recording (file/URL)** | ❌ live capture only | ❌ | ✅ any file or URL (ffmpeg + yt-dlp) | -| **Opt-in recording + consent notice** | ⚠️ partial | ❌ | ✅ off by default, one-time consent | -| **At-rest encryption** | ☁️ server-side | ❌ | ✅ vault: Argon2id + XChaCha20-Poly1305 | -| **Self-hosted sync w/ client-side encryption** | ❌ | ⚠️ | ✅ WebDAV + OAuth, **encrypt-before-upload** | -| **Coding-agent (MCP) server** | ✅ **cloud** MCP (notes via their servers) | ❌ | ✅ **local, loopback-only**, token-gated, adds no egress | -| **License** | Proprietary | Open source (MIT) | Open source (MIT / Apache-2.0) | -| **Cost** | Subscription | Free | Free | +| | **Granola** | **Meetily** | **WhispAssist** | +| ---------------------------------------------- | ----------------------------------------------- | --------------------------------- | ----------------------------------------------------------- | +| **Positioning** | Cloud AI notepad | Open-source self-hosted assistant | Local, Windows-native assistant | +| **Platform** | macOS, Windows | macOS, Windows, Linux | Windows 10/11 | +| **Bot-free system-audio capture** | ✅ | ✅ | ✅ WASAPI loopback | +| **Transcription** | ☁️ cloud | ✅ local (Whisper) | ✅ local (whisper.cpp) | +| **NPU / GPU auto-acceleration** | n/a (cloud) | ⚠️ CPU/GPU, manual | ✅ **NPU→NVIDIA→AMD→Intel→CPU** ladder, one binary | +| **Speaker diarization** | ✅ cloud | ⚠️ limited | ✅ offline (sherpa-onnx) | +| **Summaries / notes AI** | ☁️ cloud LLM | ✅ local (Ollama) / BYO | ✅ local (Ollama, localhost **or LAN**) + optional hosted | +| **Interface localization (UI language)** | ⚠️ English-only UI (multilingual transcription) | ⚠️ English-only UI (more planned) | ✅ **20 languages** + RTL (Arabic/Urdu) | +| **Default data egress** | ☁️ audio + notes to cloud | 🔒 local (cloud optional) | 🔒 **none** — everything off by default, allowlist-enforced | +| **Calendar / Outlook `.pst` context** | ✅ cloud calendar | ❌ | ✅ local `.pst` import + **auto-record on events** | +| **Import an existing recording (file/URL)** | ❌ live capture only | ❌ | ✅ any file or URL (ffmpeg + yt-dlp) | +| **Opt-in recording + consent notice** | ⚠️ partial | ❌ | ✅ off by default, one-time consent | +| **At-rest encryption** | ☁️ server-side | ❌ | ✅ vault: Argon2id + XChaCha20-Poly1305 | +| **Self-hosted sync w/ client-side encryption** | ❌ | ⚠️ | ✅ WebDAV + OAuth, **encrypt-before-upload** | +| **Coding-agent (MCP) server** | ✅ **cloud** MCP (notes via their servers) | ❌ | ✅ **local, loopback-only**, token-gated, adds no egress | +| **License** | Proprietary | Open source (MIT) | Open source (MIT / Apache-2.0) | +| **Cost** | Subscription | Free | Free | Comparison reflects each project's public positioning as of mid-2026. Granola now ships an MCP server too, but it is **cloud-hosted** — an agent reaching it pulls your notes through @@ -56,29 +59,36 @@ accelerated, zero-egress-by-default** option: it exploits the NPU/GPU in modern everything on the device unless you opt in, and adds Windows-specific context (Outlook) and a coding-agent handoff. -## What's built (v0.5.2) +## What's built (v0.7.3) -- **Bot-free capture — now both sides, with a live dual meter** — WASAPI loopback records the - system mix (all participants), and an optional **microphone** path captures your own voice, - mixed into both the live transcript and the saved recording. While recording, a **level meter - overlays the system and microphone signals in two colours** so you can see both sides are being - picked up. Pick a specific output/input or turn the mic off in **Settings ▸ Hardware**. No +- **Bot-free capture — now both sides, with a live dual meter and mic mute** — WASAPI loopback + records the system mix (all participants), and an optional **microphone** path captures your + own voice, mixed into both the live transcript and the saved recording. While recording, a + **level meter overlays the system and microphone signals in two colours** so you can see both + sides are being picked up, and the **M key (or the mic button) mutes/unmutes your mic + mid-meeting**. Pick a specific output/input or turn the mic off in **Settings ▸ Hardware**. No meeting bot, no per-app plumbing. - **Local transcription with a hardware ladder — and a fluid live transcript** — whisper.cpp via `whisper-rs` on CPU; the **Intel NPU** via ONNX Runtime + OpenVINO; **GPU via Vulkan** (a single binary that runs on NVIDIA, AMD, and Intel). WA detects the hardware, picks the best backend - (**NPU → NVIDIA → AMD → Intel → CPU**), and shows it in the UI. The **live transcript streams a - growing line that refreshes ~once a second and commits at natural pauses** — words appear as - they're spoken instead of in fixed multi-second blocks, so sentences aren't chopped across - lines. -- **Speaker diarization** — `sherpa-onnx` (pyannote segmentation + speaker-embedding clustering), - fully offline. Install the two diarization models in **Settings ▸ Hardware** and finished - recordings are split by speaker (Speaker 1, Speaker 2, …); the microphone speaker is - auto-labelled from a short voiceprint. -- **Import an existing recording** — add a meeting from a **local audio/video file or a URL** - (YouTube, a streaming page, or a direct media link). WhispAssist transcribes and diarizes it - just like a live recording. Uses **`ffmpeg`** (and **`yt-dlp`** for URLs), which you install - yourself — neither is bundled. + (**NPU → NVIDIA → AMD → Intel → CPU**), and shows it in the UI. On NPU machines the engine + **splits Whisper across accelerators — encoder on the NPU, KV-cache decoder on the Intel iGPU — + so transcription leaves the CPU near-idle**. The **live transcript streams a growing line that + refreshes ~once a second and commits at natural pauses** — words appear as they're spoken + instead of in fixed multi-second blocks, so sentences aren't chopped across lines. +- **Speaker diarization that knows who _you_ are** — `sherpa-onnx` (pyannote segmentation + + speaker-embedding clustering), fully offline. Install the two diarization models in + **Settings ▸ Hardware** and recordings are split by speaker (You, Speaker 2, …). With the mic + enabled, your voice is recorded on its **own channel**, so your lines are labelled **You** live + — from the first pass, not just after the meeting — and attributed by per-channel voice + activity rather than clustering guesswork. Renaming a speaker updates the transcript **and the + generated notes**. +- **Import an existing recording — in the background, with a progress tracker** — add a meeting + from a **local audio/video file or a URL** (YouTube, a streaming page, or a direct media link); + it transcribes quietly in the background behind a four-step tracker (_Transcode → Transcribe → + Identify speakers → Finalize_) while you keep working. Uses **`ffmpeg`** (and **`yt-dlp`** for + URLs), which you install yourself — neither is bundled, and one-click download buttons are in + the Import dialog. - **Notes & summaries** — Markdown notes; local-LLM summaries via **Ollama** on `localhost` **or a private LAN endpoint** (RFC-1918), with a full advanced-parameter panel (system prompt, `think`, `keep_alive`, `num_ctx`, sampling/repetition/mirostat, etc.). @@ -90,8 +100,10 @@ coding-agent handoff. recordings are decrypted **in memory on the fly** (nothing plaintext is written to disk). Recordings are 16-bit for roughly half the size, and an accidental recording can be **cancelled** (audio + transcript deleted). -- **Notes, summaries & AI tags** — Markdown notes with an **Editor/Preview** toggle; local-LLM - summaries and one-click **tag generation** with a chip-based tag editor and tag filtering. +- **Notes, summaries & AI tags** — Markdown notes with an **Editor/Preview** toggle, a formatting + toolbar, and **slash commands** (`/todo`, `/h1`, `/quote`, …); **AI-enhance** turns rough notes + into clean structured Markdown via your local LLM (with one-step Undo); local-LLM summaries and + one-click **tag generation** with a chip-based tag editor and tag filtering. - **At-rest encryption vault** — Argon2id key derivation + XChaCha20-Poly1305; transcripts, notes, summaries, and recordings sealed on disk; startup unlock gate; keys zeroized on lock. - **Self-hosted sync (optional, off by default)** — WebDAV (Nextcloud, ownCloud, Cloudreve, @@ -103,7 +115,11 @@ coding-agent handoff. - **Outlook `.pst` / calendar context, with auto-record** — import events and attendees from a local Outlook `.pst` backup (read-only, range-limited, de-duplicated, with cleanup), attach meetings to events, and — opt-in — **auto-start recording when a calendar event begins** while - the app is open (a one-shot timer, no background polling). + the app is open (a one-shot timer, no background polling). The import shells out to `readpst` + (libpst) — **Settings ▸ Calendar** has a one-click download button for it. +- **Close to tray** — closing the window keeps WhispAssist running in the background; reopen it + from the tray icon, quit fully from the tray menu. On by default, toggleable in + **Settings ▸ Recording**. - **Local MCP server for coding-agent handoff** — hand meeting context to your own coding agents (Claude, Codex, Copilot, OpenCode) over a **loopback-only, token-gated** MCP server that is off by default, scope-limited, audited, and **adds no egress** — data leaves only via the agent's @@ -116,9 +132,10 @@ coding-agent handoff. Sinhala (සිංහල), Spanish — Spain & Mexico (Español), Swedish (Svenska), Tamil (தமிழ்), and Urdu (اردو). Selecting Arabic or Urdu flips the interface to **right-to-left**; any untranslated string falls back to English rather than showing a raw key. Adding a language is a one-file job. -- **One universal installer** — a single signed MSI and NSIS `-setup.exe` that covers every - machine: Vulkan for all GPUs, the Intel NPU path (+ DirectML fallback), and CPU. The Vulkan - loader is bundled so it launches even on machines without a GPU driver. +- **One universal installer** — a single MSI and NSIS `-setup.exe` that covers every machine: + Vulkan for all GPUs, the Intel NPU path (+ DirectML fallback), and CPU. The Vulkan loader is + bundled so it launches even on machines without a GPU driver. Every release ships **SHA-256 + checksums** in its release notes. **In progress:** Microsoft Graph calendar (cloud calendar via OAuth) and an optional CUDA (NVIDIA-only) build variant. @@ -132,7 +149,9 @@ during setup). 1. Download the latest **`WhispAssist__x64_en-US.msi`** from the [releases page](https://git.dou.bet/iamdoubz/WhispAssist/releases). 2. Double-click the `.msi` and follow the prompts, accepting the UAC prompt to install. The - installer is signed; if Windows SmartScreen appears, choose **More info → Run anyway**. + installers are currently unsigned, so if Windows SmartScreen appears, choose + **More info → Run anyway** (verify the download against the SHA-256 checksums in the release + notes first). 3. Launch **WhispAssist** from the Start menu. On first run WA detects your hardware and picks the best transcription backend @@ -150,11 +169,13 @@ for silent install, custom install location, and presetting defaults with a `wa- ## Optional dependencies -If you do not have these installed, WhispAssist will still work, but some features will be unavailable. +If you do not have these installed, WhispAssist will still work, but some features will be +unavailable. The app links to each one where it's needed: `ffmpeg`/`yt-dlp` in the Import dialog, +`libpst` in **Settings ▸ Calendar**. -- [yt-dlp](https://github.com/yt-dlp/yt-dlp/releases) -- [ffmpeg](https://www.ffmpeg.org/download.html#build-windows) -- [libpst](https://sourceforge.net/projects/ezwinports/files/libpst-0.6.63-w32-bin.zip/) +- [yt-dlp](https://github.com/yt-dlp/yt-dlp/releases) — importing from URLs +- [ffmpeg](https://www.ffmpeg.org/download.html#build-windows) — importing files/URLs +- [libpst](https://sourceforge.net/projects/ezwinports/files/libpst-0.6.63-w32-bin.zip/) — Outlook `.pst` calendar import ## Technology diff --git a/RELEASE_NOTES_0.7.3.md b/RELEASE_NOTES_0.7.3.md new file mode 100644 index 0000000..7b11562 --- /dev/null +++ b/RELEASE_NOTES_0.7.3.md @@ -0,0 +1,94 @@ +# WhispAssist v0.7.3 + +**Privacy-first, Windows-native meeting assistant — everything on-device, nothing leaves unless you say so.** + +A performance release: on Intel Core Ultra machines, transcription now leaves the **CPU nearly +idle**. The Whisper encoder keeps running on the **NPU**; the decoder — previously the dominant +CPU cost — now decodes with a **KV cache** and runs on the **Intel iGPU**. Nothing here changes +the privacy posture: the one new download is the upgraded transcription model, fetched from the +same source as before. + +One universal installer (**MSI** and **NSIS**) covers every machine: **Vulkan** for all GPUs +(NVIDIA/AMD/Intel), the Intel **NPU** (OpenVINO), a **DirectML** fallback, and **CPU**. + +--- + +## ⚡ Faster, cooler transcription on NPU systems + +### KV-cache decoding +The NPU engine previously re-processed the entire token sequence for every new token it +generated. It now uses the upgraded *merged* Whisper decoder and caches attention state between +steps, so each step does a small, constant amount of work. This benefits **every** machine using +the NPU/DirectML engine, whatever device the decoder runs on. + +### Decoder on the Intel iGPU +On NPU systems the decoder now runs on the **Intel iGPU** via the same OpenVINO runtime already +installed for the NPU — no new runtime download. Division of labor while transcribing: + +- **Encoder** → NPU (the short periodic spikes on Task Manager's NPU graph) +- **Decoder** → Intel iGPU +- **CPU** → audio plumbing only + +Measured on a Core Ultra 5 135U (base.en): decoding on the iGPU ran **~4.7× faster** than the +same KV-cache decode on the CPU (1.29 s vs 6.10 s for a 19 s clip), with identical transcripts — +and that work no longer occupies the CPU at all. + +**Fallbacks:** no usable Intel iGPU → the decoder automatically drops back to the CPU (still +with the KV-cache speedup), and the log says so. Machines with an NVIDIA/AMD GPU can keep using +the Vulkan backend (Settings ▸ Hardware), which runs the whole model on that GPU. The startup +log names both devices: `transcription engine: ONNX base.en (encoder EP: OpenVINO/NPU, decoder +EP: OpenVINO/GPU)`. + +### Model update (automatic, one-time) +The engine's decoder artifact changed to the merged export (`decoder_model_merged.onnx`, +~208 MB, same trusted source). WhispAssist fetches it automatically in the background on first +launch after updating; until it arrives, transcription simply uses the whisper.cpp engine, as on +a fresh install. The superseded decoder file is cleaned up automatically. + +--- + +## ✨ Small addition + +**Settings ▸ Calendar** now has a one-click **libpst (readpst)** download button — the same style +as the Import dialog's `ffmpeg`/`yt-dlp` buttons — linking to the ezwinports Windows build. +`.pst` calendar import shells out to `readpst`, which isn't bundled; the button just opens the +download page in your browser. + +--- + +## 📦 Install + +**Requirements:** Windows 10 or 11 (x64). WhispAssist needs **WebView2** (preinstalled on Windows 11; +the installer fetches it on Windows 10). Importing from a file/URL additionally needs **`ffmpeg`** +(and **`yt-dlp`** for URLs) on your PATH — the Import dialog links to both. + +1. Download **`WhispAssist_0.7.3_x64_en-US.msi`** (or the NSIS **`WhispAssist_0.7.3_x64-setup.exe`**). +2. Run it and accept the UAC prompt. If SmartScreen appears, choose **More info → Run anyway**. +3. Launch **WhispAssist** from the Start menu. + +On first run WA picks the best transcription backend (**NPU → NVIDIA → AMD → Intel → CPU**). It runs +without admin rights, does **not** add itself to startup unless you opt in, and keeps all data under +`%LOCALAPPDATA%\WhispAssist`. + +Deploying to many machines? See [`docs/enterprise-deployment.md`](docs/enterprise-deployment.md). + +## 🔐 Checksums (SHA-256) + +``` +91a32a17addc31fbc00f38966d52402ca0c5e9e7a581a0431c383ca5a60532bd WhispAssist_0.7.3_x64_en-US.msi +764b758544b075b5aefe9dde37b04e4966df528912fb884ed1be5bd57905ecdb WhispAssist_0.7.3_x64-setup.exe +``` + +Verify after download: + +```powershell +Get-FileHash .\WhispAssist_0.7.3_x64_en-US.msi -Algorithm SHA256 +``` + +--- + +## Privacy, unchanged +Everything optional is **off by default**. With nothing configured, WhispAssist makes **no content +egress at all**. Recording is opt-in; sync/AI credentials live only in the OS credential store; the +MCP server is loopback-only and adds no egress; the deployment file never carries secrets. The +reachable-host allowlist is derived from your settings and enforced in the core. diff --git a/docs/adr/0004-hardware-acceleration.md b/docs/adr/0004-hardware-acceleration.md index ac0b6c4..fba6d4f 100644 --- a/docs/adr/0004-hardware-acceleration.md +++ b/docs/adr/0004-hardware-acceleration.md @@ -44,3 +44,20 @@ Detection mapping: ## Revisit if Windows ML supersedes DirectML for our models, or a vendor SDK gives materially better NPU throughput than the ONNX/DirectML path. + +## Update (2026-07-16): NPU path uses OpenVINO; decoder split off the CPU + +As built (T3.4 + follow-up), the NPU tier runs on **ONNX Runtime + the OpenVINO EP** +(`device_type=NPU`), not DirectML — Intel's prebuilt OpenVINO ORT runtime is downloaded on +demand (DirectML remains the *GPU* fallback EP for AMD/Intel in non-Vulkan builds). The engine +splits Whisper across devices: + +- **encoder** (fixed shape): OpenVINO **NPU**; +- **decoder** (autoregressive, KV-cache greedy via Optimum's merged export): OpenVINO **GPU** + (Intel iGPU, same runtime bundle) when available, else the CPU EP — + `WA_ONNX_DECODER_DEVICE=cpu|gpu` overrides. + +Rationale: the dynamic-shaped decode loop is a poor NPU fit, and pre-KV-cache it re-ran the full +token prefix every step on the CPU — the dominant CPU cost while transcribing. With the merged +decoder + iGPU placement, sustained CPU during transcription is near-idle on NPU+iGPU systems. +The engine logs both EPs at load (`transcription engine: ONNX …`). diff --git a/package.json b/package.json index 5de12bc..1f12387 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "whispassist", "private": true, - "version": "0.7.2", + "version": "0.7.3", "type": "module", "description": "Privacy-first, fully local Windows meeting assistant.", "license": "MIT OR Apache-2.0", diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index b036b3d..c4dd311 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -6088,7 +6088,7 @@ dependencies = [ [[package]] name = "whispassist" -version = "0.7.2" +version = "0.7.3" dependencies = [ "argon2", "async-trait", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 3db30ed..d13135d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "whispassist" -version = "0.7.2" +version = "0.7.3" description = "Privacy-first, fully local Windows meeting assistant" authors = ["WhispAssist contributors"] license = "MIT OR Apache-2.0" diff --git a/src-tauri/src/audio/mod.rs b/src-tauri/src/audio/mod.rs index 0d71224..c11216b 100644 --- a/src-tauri/src/audio/mod.rs +++ b/src-tauri/src/audio/mod.rs @@ -746,10 +746,9 @@ fn capture_loop( let rate = session.format.get_samplespersec() as usize; // Advance the timeline by the wall-clock gap (capped), but // never fewer frames than the mic we must write. - let gap_frames = (elapsed.min(SILENCE_TICK_MAX).as_micros() as usize - * rate - / 1_000_000) - .max(mic.len()); + let gap_frames = + (elapsed.min(SILENCE_TICK_MAX).as_micros() as usize * rate / 1_000_000) + .max(mic.len()); let bpf = session.format.get_nchannels() as usize * (session.format.get_bitspersample() as usize / 8); let silence = vec![0u8; gap_frames * bpf]; @@ -939,22 +938,26 @@ fn write_wav_split( if channels == 0 { return Ok(0); } - let (bytes_per_sample, decode): (usize, fn(&[u8]) -> f32) = match (sample_type, format.get_bitspersample()) { - (SampleType::Float, 32) => (4, |c| f32::from_le_bytes(c.try_into().unwrap())), - (SampleType::Int, 16) => (2, |c| { - i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32 - }), - (st, bits) => { - return Err(AudioError::Device(format!( - "unsupported capture format: {st} {bits}-bit" - ))) - } - }; + let (bytes_per_sample, decode): (usize, fn(&[u8]) -> f32) = + match (sample_type, format.get_bitspersample()) { + (SampleType::Float, 32) => (4, |c| f32::from_le_bytes(c.try_into().unwrap())), + (SampleType::Int, 16) => (2, |c| { + i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32 + }), + (st, bits) => { + return Err(AudioError::Device(format!( + "unsupported capture format: {st} {bits}-bit" + ))) + } + }; let mut frames = 0u64; for frame in loopback_bytes.chunks_exact(bytes_per_sample * channels) { // R = loopback downmixed to mono. - let loop_mono = - frame.chunks_exact(bytes_per_sample).map(decode).sum::() / channels as f32; + let loop_mono = frame + .chunks_exact(bytes_per_sample) + .map(decode) + .sum::() + / channels as f32; // L = mic for this frame (silent if the mic underran). let mic_s = mic.get(frames as usize).copied().unwrap_or(0.0); writer @@ -1120,7 +1123,11 @@ fn read_wav_16k(path: &Path, channel: Option) -> Result, AudioEr let mono: Vec = match (spec.sample_format, spec.bits_per_sample) { (SampleFormat::Float, 32) => bytes .chunks_exact(4 * channels) - .map(|frame| reduce_frame(frame, 4, pick, channels, |c| f32::from_le_bytes(c.try_into().unwrap()))) + .map(|frame| { + reduce_frame(frame, 4, pick, channels, |c| { + f32::from_le_bytes(c.try_into().unwrap()) + }) + }) .collect(), (SampleFormat::Int, 16) => bytes .chunks_exact(2 * channels) @@ -1201,8 +1208,8 @@ pub fn write_wav_mono_16k(path: &Path, samples: &[f32]) -> Result<(), AudioError bits_per_sample: 32, sample_format: SampleFormat::Float, }; - let mut writer = - WavWriter::create(path, spec).map_err(|e| AudioError::Capture(format!("write wav: {e}")))?; + let mut writer = WavWriter::create(path, spec) + .map_err(|e| AudioError::Capture(format!("write wav: {e}")))?; for &s in samples { writer .write_sample(s) @@ -1338,7 +1345,10 @@ mod tests { fn voice_sample_skips_silence_and_keeps_speech() { let vs = VoiceSample::new(16_000); vs.push(&[0.0; 4_000]); // silence: below the energy floor - assert!(vs.samples().is_empty(), "silence must not fill the voiceprint"); + assert!( + vs.samples().is_empty(), + "silence must not fill the voiceprint" + ); vs.push(&[0.2; 4_000]); // speech-level audio assert_eq!(vs.samples().len(), 4_000, "speech-level audio is collected"); } diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 11867ee..e7dc258 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -194,17 +194,17 @@ fn load_transcriber( match OnnxTranscriber::load(&dir, backend, language) { Ok(t) => { // The definitive "is the accelerator actually engaged" line - // (visible with `npm run tauri dev`): the encoder runs on - // the EP below; the autoregressive decoder always runs on - // the CPU EP, which is why Task Manager shows only short - // periodic NPU spikes alongside sustained CPU load. + // (visible with `npm run tauri dev`): encoder EP from the + // accel path, decoder EP from wherever its session landed + // (OpenVINO/GPU on NPU systems, CPU otherwise/on fallback). tracing::info!( - "transcription engine: ONNX {} (encoder EP: {}, decoder: CPU)", + "transcription engine: ONNX {} (encoder EP: {}, decoder EP: {})", onnx_models::DEFAULT_ONNX_MODEL, match path { AccelPath::OnnxOpenVino => "OpenVINO/NPU", _ => "DirectML/GPU", - } + }, + t.decoder_ep() ); return Ok((Box::new(t), backend)); } @@ -347,7 +347,9 @@ async fn attribute_split( spans.sort_by_key(|s| s.start_ms); // Uniform naming: "You" -> "You", far speakers -> "Speaker 2", "Speaker 3"…. let labels = crate::diarization::voiceprint::first_appearance_order(&spans); - Some(crate::diarization::voiceprint::build_name_map(&labels, "You")) + Some(crate::diarization::voiceprint::build_name_map( + &labels, "You", + )) } /// The distinct speakers seen in `segments` so far, in first-appearance order, @@ -655,28 +657,23 @@ pub async fn start_recording( .lock() .unwrap_or_else(|e| e.into_inner()) .clone(); - if attribute_split( - diarizer.clone(), - wav_path_for_diar.clone(), - &mut snapshot, - ) - .await - .map(|auto_names| { - let mut names = names_for_diar.lock().unwrap_or_else(|e| e.into_inner()); - // Never overwrite a name already set (user rename or a - // prior pass) — same guard as the post-stop pass. - for (label, name) in auto_names { - names.entry(label).or_insert(name); - } - }) - .is_none() + if attribute_split(diarizer.clone(), wav_path_for_diar.clone(), &mut snapshot) + .await + .map(|auto_names| { + let mut names = + names_for_diar.lock().unwrap_or_else(|e| e.into_inner()); + // Never overwrite a name already set (user rename or a + // prior pass) — same guard as the post-stop pass. + for (label, name) in auto_names { + names.entry(label).or_insert(name); + } + }) + .is_none() { continue; // pass failed (already logged); retry next tick } - let relabeled: HashMap = snapshot - .into_iter() - .map(|s| (s.id, s.speaker)) - .collect(); + let relabeled: HashMap = + snapshot.into_iter().map(|s| (s.id, s.speaker)).collect(); let mut segs = segments_for_diar.lock().unwrap_or_else(|e| e.into_inner()); // Snapshot prior labels so only segments whose speaker // actually changed this pass get re-emitted (ids are stable; @@ -891,7 +888,8 @@ pub async fn stop_recording( if already_named { continue; } - if let Err(e) = state.store.rename_speaker(&meeting_id, &label, &name).await { + if let Err(e) = state.store.rename_speaker(&meeting_id, &label, &name).await + { tracing::warn!("failed to persist auto speaker name: {e}"); continue; } @@ -1757,8 +1755,11 @@ fn pick_realtime_recommendation( .max_by(|a, b| { let sa = sizes.get(&a.model).copied().unwrap_or(0); let sb = sizes.get(&b.model).copied().unwrap_or(0); - sa.cmp(&sb) - .then(b.rtf.partial_cmp(&a.rtf).unwrap_or(std::cmp::Ordering::Equal)) + sa.cmp(&sb).then( + b.rtf + .partial_cmp(&a.rtf) + .unwrap_or(std::cmp::Ordering::Equal), + ) }) .map(|r| StressRecommendation { backend: r.backend.clone(), @@ -2525,10 +2526,17 @@ async fn run_import_pipeline( }) .await; match transcode { - Ok(Ok(())) => { - emit_import_progress(&app, &meeting_id, "prepare", "done", Some(t.elapsed().as_millis()), None) + Ok(Ok(())) => emit_import_progress( + &app, + &meeting_id, + "prepare", + "done", + Some(t.elapsed().as_millis()), + None, + ), + Ok(Err(e)) => { + return fail_import(&app, &store, &meeting_id, "prepare", &e.to_string()).await } - Ok(Err(e)) => return fail_import(&app, &store, &meeting_id, "prepare", &e.to_string()).await, Err(e) => return fail_import(&app, &store, &meeting_id, "prepare", &e.to_string()).await, } @@ -2552,11 +2560,22 @@ async fn run_import_pipeline( .await; let (mut segments, resolved_language) = match transcribed { Ok(Ok(v)) => { - emit_import_progress(&app, &meeting_id, "transcribe", "done", Some(t.elapsed().as_millis()), None); + emit_import_progress( + &app, + &meeting_id, + "transcribe", + "done", + Some(t.elapsed().as_millis()), + None, + ); v } - Ok(Err(e)) => return fail_import(&app, &store, &meeting_id, "transcribe", &e.to_string()).await, - Err(e) => return fail_import(&app, &store, &meeting_id, "transcribe", &e.to_string()).await, + Ok(Err(e)) => { + return fail_import(&app, &store, &meeting_id, "transcribe", &e.to_string()).await + } + Err(e) => { + return fail_import(&app, &store, &meeting_id, "transcribe", &e.to_string()).await + } }; // Phase 3: diarize — one pass if the models are installed, else every line @@ -2578,7 +2597,14 @@ async fn run_import_pipeline( Err(e) => tracing::warn!("import diarization task failed: {e}"), } } - emit_import_progress(&app, &meeting_id, "diarize", "done", Some(t.elapsed().as_millis()), None); + emit_import_progress( + &app, + &meeting_id, + "diarize", + "done", + Some(t.elapsed().as_millis()), + None, + ); // Phase 4: finalize — persist segments, notes, seal audio. emit_import_progress(&app, &meeting_id, "finalize", "active", None, None); @@ -2626,7 +2652,14 @@ async fn run_import_pipeline( } } } - emit_import_progress(&app, &meeting_id, "finalize", "done", Some(t.elapsed().as_millis()), None); + emit_import_progress( + &app, + &meeting_id, + "finalize", + "done", + Some(t.elapsed().as_millis()), + None, + ); let _ = app.emit( "transcript://finalized", @@ -2643,7 +2676,9 @@ async fn fail_import( phase: &str, message: &str, ) { - let _ = store.set_meeting_status(&meeting_id.to_string(), "error").await; + let _ = store + .set_meeting_status(&meeting_id.to_string(), "error") + .await; emit_import_progress(app, meeting_id, phase, "error", None, Some(message)); } @@ -2926,8 +2961,8 @@ fn fold_wav(wav: &[u8], out_channels: u16) -> Option> { (hound::SampleFormat::Int, 16) => { let s: Vec = reader.samples::().map_while(Result::ok).collect(); for frame in s.chunks(ch) { - let avg = (frame.iter().map(|&x| x as i32).sum::() - / frame.len() as i32) as i16; + let avg = + (frame.iter().map(|&x| x as i32).sum::() / frame.len() as i32) as i16; for _ in 0..out_channels { writer.write_sample(avg).ok()?; } @@ -3125,10 +3160,7 @@ pub async fn bulk_export_meetings( /// so re-importing never collides with existing meetings. Returns the count /// imported. #[tauri::command] -pub async fn import_meeting_bundle( - state: State<'_, AppState>, - dir: String, -) -> WaResult { +pub async fn import_meeting_bundle(state: State<'_, AppState>, dir: String) -> WaResult { let root = PathBuf::from(&dir); let mut bundles: Vec = Vec::new(); if root.join("meeting.json").exists() { @@ -3977,7 +4009,11 @@ only, with no preamble or commentary."; let notes = notes.trim(); let user = format!( "My rough notes:\n{}\n\nTranscript:\n{}", - if notes.is_empty() { "(none yet)" } else { notes }, + if notes.is_empty() { + "(none yet)" + } else { + notes + }, prompt.transcript ); provider @@ -4895,8 +4931,8 @@ pub(crate) fn spawn_auto_sync( let app = app.clone(); tauri::async_runtime::spawn(async move { match enqueue_meeting_sync(store.as_ref(), &meeting_id, None, true).await { - Ok(0) => {} // nothing changed - Ok(_) => pump_sync(&app, store.as_ref()).await, // upload the changes + Ok(0) => {} // nothing changed + Ok(_) => pump_sync(&app, store.as_ref()).await, // upload the changes Err(e) => tracing::warn!("auto-sync enqueue failed: {e:?}"), } }); diff --git a/src-tauri/src/deploy.rs b/src-tauri/src/deploy.rs index 1b7e5e0..036db66 100644 --- a/src-tauri/src/deploy.rs +++ b/src-tauri/src/deploy.rs @@ -26,7 +26,11 @@ const AUTO_DOWNLOAD_KEY: &str = "auto_download_model"; fn candidate_paths() -> Vec { let mut paths = Vec::new(); if let Ok(program_data) = std::env::var("ProgramData") { - paths.push(PathBuf::from(program_data).join("WhispAssist").join("wa-defaults.ini")); + paths.push( + PathBuf::from(program_data) + .join("WhispAssist") + .join("wa-defaults.ini"), + ); } if let Ok(exe) = std::env::current_exe() { if let Some(dir) = exe.parent() { @@ -121,7 +125,8 @@ fn coerce(value: &str) -> Value { } fn truthy(v: &Value) -> bool { - matches!(v, Value::Bool(true)) || matches!(v, Value::String(s) if s.eq_ignore_ascii_case("true")) + matches!(v, Value::Bool(true)) + || matches!(v, Value::String(s) if s.eq_ignore_ascii_case("true")) } /// Defense in depth: never seed anything that smells like a credential. diff --git a/src-tauri/src/diarization/mod.rs b/src-tauri/src/diarization/mod.rs index 3f8e3cf..f278466 100644 --- a/src-tauri/src/diarization/mod.rs +++ b/src-tauri/src/diarization/mod.rs @@ -95,7 +95,12 @@ pub fn assign_split( seg.speaker = "You".to_string(); } else if let Some(span) = far_spans .iter() - .map(|sp| (overlap(seg.start_ms, seg.end_ms, sp.start_ms, sp.end_ms), sp)) + .map(|sp| { + ( + overlap(seg.start_ms, seg.end_ms, sp.start_ms, sp.end_ms), + sp, + ) + }) .filter(|(o, _)| *o > 0) .max_by_key(|(o, _)| *o) .map(|(_, sp)| sp) @@ -298,7 +303,12 @@ mod overlap_tests { fn split_keeps_the_prior_label_when_both_channels_are_silent() { let mut segments = vec![segment(5000, 6000)]; segments[0].speaker = "S9".to_string(); - assign_split(&mut segments, &[(0, 1000)], &[(0, 1000)], &[span(0, 1000, "S1")]); + assign_split( + &mut segments, + &[(0, 1000)], + &[(0, 1000)], + &[span(0, 1000, "S1")], + ); assert_eq!(segments[0].speaker, "S9"); } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 2840cca..cdedaae 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -117,7 +117,11 @@ fn fatal_startup_error(body: &str) -> ! { MessageBoxW, MB_ICONERROR, MB_OK, MB_SYSTEMMODAL, }; // Null-terminated UTF-16 for the Win32 wide API. - let to_wide = |s: &str| s.encode_utf16().chain(std::iter::once(0)).collect::>(); + let to_wide = |s: &str| { + s.encode_utf16() + .chain(std::iter::once(0)) + .collect::>() + }; let text = to_wide(body); let caption = to_wide("WhispAssist can't start"); // SAFETY: both buffers are null-terminated and outlive the call. @@ -228,13 +232,17 @@ pub fn run() { ) .await { - tracing::warn!("deploy auto-download of model failed: {e:?}"); + tracing::warn!( + "deploy auto-download of model failed: {e:?}" + ); } }); } } Err(e) => { - tracing::error!("first-run deploy seeding failed to write settings: {e:?}") + tracing::error!( + "first-run deploy seeding failed to write settings: {e:?}" + ) } } } diff --git a/src-tauri/src/media.rs b/src-tauri/src/media.rs index d4d2e46..9024dc8 100644 --- a/src-tauri/src/media.rs +++ b/src-tauri/src/media.rs @@ -21,10 +21,7 @@ pub enum MediaError { #[error("{0} isn't installed or on PATH — install it and try again")] ToolMissing(&'static str), #[error("{tool} failed: {message}")] - Failed { - tool: &'static str, - message: String, - }, + Failed { tool: &'static str, message: String }, #[error("io error: {0}")] Io(#[from] std::io::Error), } @@ -60,7 +57,11 @@ fn run(program: &'static str, cmd: &mut Command) -> Result<(), MediaError> { if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); let stdout = String::from_utf8_lossy(&output.stdout); - let text = if stderr.trim().is_empty() { stdout } else { stderr }; + let text = if stderr.trim().is_empty() { + stdout + } else { + stderr + }; let tail: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect(); let start = tail.len().saturating_sub(6); return Err(MediaError::Failed { @@ -145,12 +146,7 @@ mod tests { fn import_reports_a_missing_local_file_without_touching_a_tool() { let dir = std::env::temp_dir().join(format!("wa-media-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&dir).unwrap(); - let err = import_to_wav( - "no-such-file.mp4", - &dir.join("out.wav"), - &dir, - ) - .unwrap_err(); + let err = import_to_wav("no-such-file.mp4", &dir.join("out.wav"), &dir).unwrap_err(); assert!(matches!(err, MediaError::Failed { tool: "import", .. })); let _ = std::fs::remove_dir_all(&dir); } diff --git a/src-tauri/src/storage/mod.rs b/src-tauri/src/storage/mod.rs index c9c93f1..52d0e60 100644 --- a/src-tauri/src/storage/mod.rs +++ b/src-tauri/src/storage/mod.rs @@ -1260,13 +1260,15 @@ impl Store for SqliteStore { started_at: i64, ended_at: Option, ) -> Result<(), StoreError> { - sqlx::query("UPDATE meetings SET started_at = ?, ended_at = ?, updated_at = ? WHERE id = ?") - .bind(started_at) - .bind(ended_at) - .bind(now_unix()) - .bind(meeting_id) - .execute(&self.pool) - .await?; + sqlx::query( + "UPDATE meetings SET started_at = ?, ended_at = ?, updated_at = ? WHERE id = ?", + ) + .bind(started_at) + .bind(ended_at) + .bind(now_unix()) + .bind(meeting_id) + .execute(&self.pool) + .await?; Ok(()) } diff --git a/src-tauri/src/transcription/mod.rs b/src-tauri/src/transcription/mod.rs index 5462a83..a307964 100644 --- a/src-tauri/src/transcription/mod.rs +++ b/src-tauri/src/transcription/mod.rs @@ -588,7 +588,10 @@ mod tests { feed_into(&mut s, &step, &d2, &mut out); assert!(out[3].interim && out[3].text == "next sentence"); assert_ne!(out[3].id, out[2].id); - assert!(out[3].start_ms >= 3000, "starts after the 3 committed steps"); + assert!( + out[3].start_ms >= 3000, + "starts after the 3 committed steps" + ); } #[test] @@ -612,7 +615,10 @@ mod tests { assert!(out.last().unwrap().interim, "still growing before flush"); flush_into(&mut s, &decode, &mut out); let last = out.last().unwrap(); - assert!(!last.interim && last.text == "partial", "flush finalizes it"); + assert!( + !last.interim && last.text == "partial", + "flush finalizes it" + ); } #[test] diff --git a/src-tauri/src/transcription/npu.rs b/src-tauri/src/transcription/npu.rs index b6fc256..6b0fbce 100644 --- a/src-tauri/src/transcription/npu.rs +++ b/src-tauri/src/transcription/npu.rs @@ -7,21 +7,24 @@ //! //! Split of work, validated by the T3.4 spike (encoder ~3.6× faster on NPU): //! - **encoder** (fixed `[1,80,3000]` shape) runs on the **accelerator** (NPU or -//! GPU) — the expensive graph, and the fixed shape accelerators want; -//! - **decoder** (dynamic, autoregressive) runs greedy on the **CPU** EP — a -//! dynamic KV loop is a poor accelerator fit and the cheap half anyway. -//! -//! ponytail: greedy, no-KV-cache decode (re-feeds the full token prefix each -//! step). Correct and simple; windows are short so the token count is small. -//! Add `decoder_with_past` only if the decoder shows up in a profile. +//! DirectML GPU) — the expensive graph, and the fixed shape accelerators want; +//! - **decoder** (dynamic, autoregressive) decodes greedily **with a KV cache** +//! (Optimum's merged export: `use_cache_branch` + `past_key_values`), so each +//! step feeds one token instead of re-running the whole prefix. On the NPU +//! path the decoder session itself goes to the **OpenVINO GPU EP** (Intel +//! iGPU, same runtime bundle) so sustained CPU stays near-idle; it falls back +//! to the CPU EP when no iGPU is usable, and `WA_ONNX_DECODER_DEVICE=cpu|gpu` +//! overrides the choice. DirectML backends keep the CPU decoder — the DML EP +//! handles per-step-growing KV shapes poorly. use super::{AudioWindow, SegmentSink, Transcriber, TrxError}; use crate::models::{BackendId, TranscriptSegment}; use ort::execution_providers::{ DirectMLExecutionProvider, ExecutionProviderDispatch, OpenVINOExecutionProvider, }; -use ort::session::Session; -use ort::value::Tensor; +use ort::session::{Session, SessionInputValue}; +use ort::value::{Tensor, ValueType}; +use std::borrow::Cow; use std::collections::HashMap; use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; @@ -30,6 +33,36 @@ use std::sync::Mutex; /// Safety cap on generated tokens per 30 s window (Whisper's own max is 448). const MAX_NEW_TOKENS: usize = 224; +/// One KV-cache tensor pair of the merged decoder, discovered from the graph +/// at load: the `past_key_values.*` input, its matching `present.*` output, +/// whether it's decoder self-attention (refreshed every step) or encoder +/// cross-attention (computed once on the first step, then passed through +/// untouched by the cache branch), and the dummy shape to feed before any +/// past exists (see [`dummy_past_dims`]). +struct KvSlot { + past: String, + present: String, + self_attention: bool, + dummy_dims: Vec, +} + +/// `"past_key_values.3.decoder.key"` → `"present.3.decoder.key"` — Optimum's +/// fixed naming convention for the merged decoder's cache I/O. +fn present_name_for(past: &str) -> String { + format!("present.{}", past.trim_start_matches("past_key_values.")) +} + +/// The "no past yet" shape for a KV input: batch → 1, any other dynamic (−1) +/// dim → **1** (a dummy length-1 past — ORT's raw-data tensor creation rejects +/// zero-length dims, and the no-cache branch never reads the values anyway; +/// same dummy Optimum's own runner feeds), static dims kept. +fn dummy_past_dims(dims: &[i64]) -> Vec { + dims.iter() + .enumerate() + .map(|(i, &d)| if i == 0 || d < 0 { 1 } else { d as usize }) + .collect() +} + pub struct OnnxTranscriber { encoder: Mutex, decoder: Mutex, @@ -38,6 +71,13 @@ pub struct OnnxTranscriber { dec_ids: String, dec_hidden: String, dec_logits: String, + /// The merged decoder's KV-cache plumbing (see [`KvSlot`]) plus its + /// branch selector input. + kv_slots: Vec, + dec_use_cache: String, + /// Which EP the decoder session actually landed on ("OpenVINO/GPU" or + /// "CPU") — surfaced so `load_transcriber`'s engine log tells the truth. + decoder_ep: &'static str, detok: Detok, /// `[decoder_start_token_id, ...forced_decoder_ids]` — the fixed prompt /// prefix before generation begins. @@ -120,29 +160,76 @@ impl OnnxTranscriber { let hidden = Tensor::from_array((hidden_shape, hidden_data)) .map_err(|e| TrxError::Inference(e.to_string()))?; - // Greedy decode on the CPU. + // Greedy KV-cache decode. Step 1 runs the merged graph's no-cache + // branch over the full prompt and yields every present KV; later steps + // feed a single token plus the cache, refreshing only the decoder + // self-attention slots — the cache branch passes encoder + // cross-attention KVs through untouched, so the step-1 tensors stay + // authoritative (same contract as Optimum's own runner). + // ponytail: KVs round-trip host memory each step; wire ort IoBinding + // to pin them on-device if GPU-decoder profiling shows the copies matter. + let mut kv_cache: Vec> = self + .kv_slots + .iter() + .map(|slot| { + let len = slot.dummy_dims.iter().product::(); + Tensor::from_array((slot.dummy_dims.clone(), vec![0.0f32; len])) + }) + .collect::>() + .map_err(|e| TrxError::Inference(e.to_string()))?; + let mut tokens = self.initial_tokens.clone(); + let mut cached = false; // flips true once step 1 has filled kv_cache for _ in 0..MAX_NEW_TOKENS { - let ids: Vec = tokens.clone(); + let ids: Vec = if cached { + vec![*tokens.last().unwrap_or(&self.eot)] + } else { + tokens.clone() + }; let ids_tensor = Tensor::from_array(([1usize, ids.len()], ids)) .map_err(|e| TrxError::Inference(e.to_string()))?; + let branch = Tensor::from_array(([1usize], vec![cached])) + .map_err(|e| TrxError::Inference(e.to_string()))?; - let next = { + // Presents are extracted (copied) inside the outputs' borrow scope, + // then written back into kv_cache after it ends. + let (next, fresh_kvs) = { let mut dec = self .decoder .lock() .map_err(|_| TrxError::Inference("decoder mutex poisoned".into()))?; + let mut inputs: Vec<(Cow<'_, str>, SessionInputValue<'_>)> = vec![ + (Cow::from(self.dec_ids.as_str()), ids_tensor.view().into()), + (Cow::from(self.dec_hidden.as_str()), hidden.view().into()), + (Cow::from(self.dec_use_cache.as_str()), branch.view().into()), + ]; + for (slot, kv) in self.kv_slots.iter().zip(kv_cache.iter()) { + inputs.push((Cow::from(slot.past.as_str()), kv.view().into())); + } let outputs = dec - .run(ort::inputs![ - self.dec_ids.as_str() => ids_tensor.view(), - self.dec_hidden.as_str() => hidden.view(), - ]) + .run(inputs) .map_err(|e| TrxError::Inference(e.to_string()))?; let (shape, data) = outputs[self.dec_logits.as_str()] .try_extract_tensor::() .map_err(|e| TrxError::Inference(e.to_string()))?; - self.argmax_last(shape, data) + let next = self.argmax_last(shape, data); + + let mut fresh: Vec<(usize, Vec, Vec)> = Vec::new(); + for (i, slot) in self.kv_slots.iter().enumerate() { + if slot.self_attention || !cached { + let (s, d) = outputs[slot.present.as_str()] + .try_extract_tensor::() + .map_err(|e| TrxError::Inference(e.to_string()))?; + fresh.push((i, s.iter().map(|d| *d as usize).collect(), d.to_vec())); + } + } + (next, fresh) }; + for (i, dims, data) in fresh_kvs { + kv_cache[i] = Tensor::from_array((dims, data)) + .map_err(|e| TrxError::Inference(e.to_string()))?; + } + cached = true; if next == self.eot { break; @@ -180,6 +267,11 @@ impl OnnxTranscriber { best } + /// Which EP the decoder session landed on — "OpenVINO/GPU" or "CPU". + pub fn decoder_ep(&self) -> &'static str { + self.decoder_ep + } + fn segment(&self, text: String, start_ms: u64, end_ms: u64) -> Option { if text.is_empty() { return None; @@ -252,7 +344,7 @@ impl Transcriber for OnnxTranscriber { Self::ensure_runtime_env(runtime_dll); let encoder_path = model.join("encoder_model.onnx"); - let decoder_path = model.join("decoder_model.onnx"); + let decoder_path = model.join("decoder_model_merged.onnx"); let tokenizer_path = model.join("tokenizer.json"); let gen_config_path = model.join("generation_config.json"); @@ -263,11 +355,46 @@ impl Transcriber for OnnxTranscriber { .commit_from_file(&encoder_path) .map_err(|e| TrxError::Load(e.to_string()))?; - // Decoder on the default CPU EP (dynamic shapes; poor NPU fit). - let decoder = Session::builder() - .map_err(|e| TrxError::Load(e.to_string()))? - .commit_from_file(&decoder_path) - .map_err(|e| TrxError::Load(e.to_string()))?; + // Decoder session. The KV-cache loop is dynamic-shaped and + // autoregressive — a poor NPU fit — but it needn't burn CPU either: on + // the NPU path, try the same OpenVINO runtime's GPU plugin (Intel + // iGPU) first and fall back to the default CPU EP if no GPU is usable. + // `WA_ONNX_DECODER_DEVICE=cpu|gpu` overrides (debug escape hatch, like + // WA_NPU_RUNTIME_URL). DirectML backends keep the CPU decoder — the + // DML EP handles per-step-growing KV shapes poorly. + let want_gpu = match std::env::var("WA_ONNX_DECODER_DEVICE").as_deref() { + Ok("cpu") => false, + Ok("gpu") => true, + _ => matches!(backend, BackendId::Npu), + }; + let mut decoder_ep = "CPU"; + let mut decoder = None; + if want_gpu { + let gpu = Session::builder() + .and_then(|b| { + b.with_execution_providers([OpenVINOExecutionProvider::default() + .with_device_type("GPU") + .build() + .error_on_failure()]) + }) + .and_then(|b| b.commit_from_file(&decoder_path)); + match gpu { + Ok(s) => { + decoder_ep = "OpenVINO/GPU"; + decoder = Some(s); + } + Err(e) => tracing::warn!( + "OpenVINO GPU EP unavailable for the decoder ({e}); using the CPU EP" + ), + } + } + let decoder = match decoder { + Some(s) => s, + None => Session::builder() + .map_err(|e| TrxError::Load(e.to_string()))? + .commit_from_file(&decoder_path) + .map_err(|e| TrxError::Load(e.to_string()))?, + }; let enc_input = Self::find_input(&encoder, "input_features")?; let enc_output = Self::find_output(&encoder, "last_hidden_state") @@ -275,6 +402,53 @@ impl Transcriber for OnnxTranscriber { let dec_ids = Self::find_input(&decoder, "input_ids")?; let dec_hidden = Self::find_input(&decoder, "encoder_hidden")?; let dec_logits = Self::find_output(&decoder, "logits")?; + let dec_use_cache = Self::find_input(&decoder, "use_cache_branch")?; + + // KV-cache plumbing, discovered rather than hardcoded so the layer + // count comes from the graph (whisper-base: 6 layers → 24 slots). + let mut kv_slots: Vec = Vec::new(); + for input in &decoder.inputs { + if !input.name.starts_with("past_key_values.") { + continue; + } + let dims: Vec = match &input.input_type { + ValueType::Tensor { shape, .. } => shape.to_vec(), + other => { + return Err(TrxError::Load(format!( + "KV input '{}' has non-tensor type {other:?}", + input.name + ))) + } + }; + let present = present_name_for(&input.name); + if !decoder.outputs.iter().any(|o| o.name == present) { + return Err(TrxError::Load(format!( + "decoder missing present output '{present}'" + ))); + } + kv_slots.push(KvSlot { + self_attention: input.name.contains(".decoder."), + dummy_dims: dummy_past_dims(&dims), + past: input.name.clone(), + present, + }); + } + if kv_slots.is_empty() { + return Err(TrxError::Load( + "decoder has no past_key_values inputs — expected the merged (KV-cache) export" + .into(), + )); + } + + // A pre-KV-cache install leaves the old plain decoder behind — 208 MB + // of dead weight once the merged graph loads. Best-effort removal. + let legacy = model.join("decoder_model.onnx"); + if legacy.exists() { + match std::fs::remove_file(&legacy) { + Ok(()) => tracing::info!("removed superseded decoder_model.onnx"), + Err(e) => tracing::warn!("couldn't remove superseded decoder_model.onnx: {e}"), + } + } let detok = Detok::from_tokenizer_json(&tokenizer_path)?; let gen = GenConfig::from_file(&gen_config_path)?; @@ -290,6 +464,9 @@ impl Transcriber for OnnxTranscriber { dec_ids, dec_hidden, dec_logits, + kv_slots, + dec_use_cache, + decoder_ep, detok, initial_tokens, special_floor: gen.decoder_start, @@ -457,6 +634,28 @@ mod tests { assert_eq!(d.get(&'A'), Some(&b'A')); } + #[test] + fn present_name_follows_optimum_convention() { + assert_eq!( + present_name_for("past_key_values.3.decoder.key"), + "present.3.decoder.key" + ); + assert_eq!( + present_name_for("past_key_values.0.encoder.value"), + "present.0.encoder.value" + ); + } + + #[test] + fn dummy_past_dims_pins_dynamic_axes_to_one_and_keeps_static_ones() { + // whisper-base decoder self-attention past: [batch, 8, past_seq, 64] + // with batch/past_seq dynamic — both pin to 1 (ORT rejects 0-length + // dims for raw-data tensors; the no-cache branch ignores the values). + assert_eq!(dummy_past_dims(&[-1, 8, -1, 64]), vec![1, 8, 1, 64]); + // A static trailing shape is preserved (batch still pins to 1). + assert_eq!(dummy_past_dims(&[-1, 8, 1500, 64]), vec![1, 8, 1500, 64]); + } + #[test] fn byte_decoder_covers_all_256_bytes() { let d = byte_decoder(); @@ -481,10 +680,14 @@ mod tests { return; }; let wav = std::env::var("WA_NPU_TEST_WAV").expect("set WA_NPU_TEST_WAV"); + // Surface the engine's own EP-selection logs (GPU-decoder fallback + // warnings etc.) — the test harness has no subscriber otherwise. + let _ = tracing_subscriber::fmt().with_env_filter("info").try_init(); let t0 = std::time::Instant::now(); let t = OnnxTranscriber::load(Path::new(&model_dir), BackendId::Npu, None) .expect("load NPU transcriber"); let load_ms = t0.elapsed().as_millis(); + eprintln!("[spike] decoder EP: {}", t.decoder_ep()); let t1 = std::time::Instant::now(); let segs = t.transcribe_file(Path::new(&wav)).expect("transcribe"); let infer_ms = t1.elapsed().as_millis(); diff --git a/src-tauri/src/transcription/onnx_models.rs b/src-tauri/src/transcription/onnx_models.rs index 083d935..9060066 100644 --- a/src-tauri/src/transcription/onnx_models.rs +++ b/src-tauri/src/transcription/onnx_models.rs @@ -15,9 +15,19 @@ const REPO: &str = "onnx-community/whisper-base.en"; pub const DEFAULT_ONNX_MODEL: &str = "base.en"; /// The four artifacts the `OnnxTranscriber` needs, relative to the repo root. +/// The decoder is Optimum's *merged* export — both the no-past and with-past +/// branches in one graph, selected by a `use_cache_branch` input — which is +/// what enables the KV-cache decode loop (the plain `decoder_model.onnx` +/// re-ran the whole prefix every step). Existing installs migrate for free: +/// swapping this entry flips `is_installed` to false, the startup background +/// fetch re-runs `download`, and `download` skips the three files already on +/// disk, fetching only the merged decoder. const FILES: &[(&str, &str)] = &[ ("encoder_model.onnx", "onnx/encoder_model.onnx"), - ("decoder_model.onnx", "onnx/decoder_model.onnx"), + ( + "decoder_model_merged.onnx", + "onnx/decoder_model_merged.onnx", + ), ("tokenizer.json", "tokenizer.json"), ("generation_config.json", "generation_config.json"), ]; @@ -45,7 +55,7 @@ pub fn model_files(id: &str) -> OnnxModelFiles { let dir = onnx_dir(id); OnnxModelFiles { encoder: dir.join("encoder_model.onnx"), - decoder: dir.join("decoder_model.onnx"), + decoder: dir.join("decoder_model_merged.onnx"), tokenizer: dir.join("tokenizer.json"), generation_config: dir.join("generation_config.json"), } @@ -125,6 +135,6 @@ mod tests { fn catalog_lists_the_four_required_artifacts() { assert_eq!(FILES.len(), 4); assert!(FILES.iter().any(|(n, _)| *n == "encoder_model.onnx")); - assert!(FILES.iter().any(|(n, _)| *n == "decoder_model.onnx")); + assert!(FILES.iter().any(|(n, _)| *n == "decoder_model_merged.onnx")); } } diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 98af576..09b6312 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "WhispAssist", - "version": "0.7.2", + "version": "0.7.3", "identifier": "bet.dou.whispassist", "build": { "frontendDist": "../dist", diff --git a/src/lib/i18n/en.json b/src/lib/i18n/en.json index f9a4bb0..47db52d 100644 --- a/src/lib/i18n/en.json +++ b/src/lib/i18n/en.json @@ -107,6 +107,7 @@ "settings.calendar.title": "Calendar & Outlook .pst", "settings.calendar.intro_1": "Import events and attendees from a local Outlook", "settings.calendar.intro_2": "backup — read-only, nothing is written back to the file. Nothing leaves this device.", + "settings.calendar.requires": "Needs this on your PATH (not bundled):", "settings.calendar.pst_file": ".pst file", "settings.calendar.no_file": "No file selected", "settings.calendar.browse": "Browse…", diff --git a/src/lib/views/Settings.svelte b/src/lib/views/Settings.svelte index 360716b..1c916f4 100644 --- a/src/lib/views/Settings.svelte +++ b/src/lib/views/Settings.svelte @@ -286,6 +286,11 @@ } // ---- Calendar / .pst import (T6.1/T6.2/T6.3, FR-CAL-1/2) ---- + // .pst import shells out to readpst (libpst, ADR-0008) — external like + // ffmpeg/yt-dlp, so link the same kind of download button as the Import + // dialog's. ezwinports is the maintained Windows build of libpst. + const LIBPST_URL = + "https://sourceforge.net/projects/ezwinports/files/libpst-0.6.63-w32-bin.zip/download"; let pstPath = $state(""); let pstPassword = $state(""); let eventTitleFilter = $state(""); @@ -987,7 +992,9 @@

{t("settings.hardware.stress_hint")}

{#if stressProgress}

{t("settings.hardware.stress_progress", { pair: stressProgress })}

@@ -1344,6 +1351,12 @@ {t("settings.calendar.intro_1")} .pst {t("settings.calendar.intro_2")}

+
+ {t("settings.calendar.requires")} + +