diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f3ee90e --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +runtime/*.7z filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index a4c3735..2ad5e85 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ on-device acceleration (**NPU → GPU → CPU**), labels speakers, structures th Markdown notes, and optionally augments them with a locally hosted LLM (Ollama). Audio and transcripts **never leave the machine** unless you explicitly configure a destination. -> **Status: working application (v0.1.4).** Capture, transcription (CPU / Intel NPU / Vulkan +> **Status: working application (v0.1.5).** Capture, transcription (CPU / Intel NPU / Vulkan > GPU), speaker diarization, storage + crash recovery, local-LLM summaries, opt-in recording, > at-rest encryption, and self-hosted sync are implemented and ship as signed **MSI + NSIS** > installers. Outlook `.pst`/calendar context and the coding-agent (MCP) handoff are in @@ -44,7 +44,7 @@ accelerated, zero-egress-by-default** option: it exploits the NPU/GPU in modern everything on the device unless you opt in, and adds Windows-specific context (Outlook) and a coding-agent handoff. -## What's built (v0.1.4) +## What's built (v0.1.5) - **Bot-free capture** — WASAPI loopback records the system mix (all participants) with no meeting bot and no per-app plumbing. @@ -76,6 +76,26 @@ coding-agent handoff. **In progress:** Outlook `.pst` + calendar context, the local **MCP server** that hands meeting context to your coding agents (Claude, Codex, Copilot, OpenCode), and MS Graph calendar. +## Quick start (install) + +**Requirements:** Windows 10 or 11 (x64). WhispAssist needs the **WebView2** runtime — preinstalled +on Windows 11; on Windows 10 the installer fetches it automatically (so keep an internet connection +during setup). + +1. Download the latest **`WhispAssist__x64_en-US.msi`** from the + [releases page](https://git.dou.bet/iamdoubz/WhispAssist/releases). +2. Double-click the `.msi` and follow the prompts, accepting the UAC prompt to install. The + installer is signed; if Windows SmartScreen appears, choose **More info → Run anyway**. +3. Launch **WhispAssist** from the Start menu. + +On first run WA detects your hardware and picks the best transcription backend +(**NPU → NVIDIA → AMD → Intel → CPU**). It works on your **CPU or GPU** (GPU via Vulkan) out of the +box; to use an Intel **NPU**, open **Settings ▸ Hardware** and download the one-time NPU +acceleration package. The app runs without admin rights, never adds itself to startup, and keeps all +data under `%LOCALAPPDATA%\WhispAssist`. + +Prefer the NSIS installer? Grab **`WhispAssist__x64-setup.exe`** from the same page. + ## Technology WhispAssist is a **Tauri 2** application: a small Rust core with a compiled **Svelte + TypeScript** diff --git a/package.json b/package.json index cf6c9dc..ee02251 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "whispassist", "private": true, - "version": "0.1.4", + "version": "0.1.5", "type": "module", "description": "Privacy-first, fully local Windows meeting assistant.", "license": "MIT OR Apache-2.0", diff --git a/runtime/directml.7z b/runtime/directml.7z new file mode 100644 index 0000000..add150b --- /dev/null +++ b/runtime/directml.7z @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34369222fcc1be2e72a957b868b1976a90150ba704a06c9e8992c34ee368926b +size 3744093 diff --git a/runtime/openvino.7z b/runtime/openvino.7z new file mode 100644 index 0000000..9c14459 --- /dev/null +++ b/runtime/openvino.7z @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca0be9fc52c78ee623b152f790450b3d4020c5a7ebe99d27736455b308782191 +size 23427084 diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 785e8bc..5374439 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -63,15 +63,6 @@ version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "argon2" version = "0.5.3" @@ -804,17 +795,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.118", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -948,7 +928,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "zip 0.6.6", + "zip", ] [[package]] @@ -1161,6 +1141,17 @@ dependencies = [ "libc", ] +[[package]] +name = "filetime_creation" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c25b5d475550e559de5b0c0084761c65325444e3b6c9e298af9cefe7a9ef3a5f" +dependencies = [ + "cfg-if", + "filetime", + "windows-sys 0.52.0", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -2387,6 +2378,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rust2" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ea7cee9b17915968314862afb1885c725891c6da8c4ac41a89ea4a8125b352" +dependencies = [ + "byteorder", +] + [[package]] name = "markup5ever" version = "0.38.0" @@ -2567,6 +2567,17 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nt-time" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c367e8edaff1f8a871e56343eb5e03888f6e0d1c2861880ccf4b7dc830899ed" +dependencies = [ + "chrono", + "rand 0.8.6", + "time", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4121,6 +4132,23 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sevenz-rust2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103547d6d9d23b2a2aa50d284dfe8fe534f3da38e851f67e7da088539b41a4a" +dependencies = [ + "bit-set", + "byteorder", + "crc32fast", + "filetime_creation", + "js-sys", + "lzma-rust2", + "nt-time", + "sha2", + "wasm-bindgen", +] + [[package]] name = "sha1" version = "0.10.6" @@ -5945,7 +5973,7 @@ dependencies = [ [[package]] name = "whispassist" -version = "0.1.4" +version = "0.1.5" dependencies = [ "argon2", "async-trait", @@ -5963,6 +5991,7 @@ dependencies = [ "rustfft", "serde", "serde_json", + "sevenz-rust2", "sha2", "sherpa-rs", "sqlx", @@ -5978,7 +6007,6 @@ dependencies = [ "whisper-rs", "windows 0.58.0", "zeroize", - "zip 2.4.2", ] [[package]] @@ -6873,41 +6901,12 @@ dependencies = [ "flate2", ] -[[package]] -name = "zip" -version = "2.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "flate2", - "indexmap 2.14.0", - "memchr", - "thiserror 2.0.18", - "zopfli", -] - [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] - [[package]] name = "zune-core" version = "0.5.1" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index b0d9f83..d4bfca6 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "whispassist" -version = "0.1.4" +version = "0.1.5" description = "Privacy-first, fully local Windows meeting assistant" authors = ["WhispAssist contributors"] license = "MIT OR Apache-2.0" @@ -55,7 +55,7 @@ whisper-rs = { version = "0.16", optional = true } # whisper.cpp bindin # dlopens the on-demand-downloaded runtime, so cargo compiles no C++/OpenVINO — # keeps the CPU-only build untouched (NFR-MNT-4). `rustfft` powers the log-mel # front-end (detok is hand-rolled off serde_json, no `tokenizers`/C dep). -ort = { version = "=2.0.0-rc.10", optional = true, default-features = false, features = ["load-dynamic", "openvino"] } +ort = { version = "=2.0.0-rc.10", optional = true, default-features = false, features = ["load-dynamic", "openvino", "directml"] } rustfft = { version = "6", optional = true } sherpa-rs = { version = "0.6", optional = true, default-features = false, features = ["download-binaries"] } # sherpa-onnx bindings (Phase 4, ADR-0005) tauri-plugin-dialog = "2" # native Save/choose-folder (Phase 2 export) @@ -64,8 +64,11 @@ tauri-plugin-dialog = "2" # native Save/choose # conversion service, consistent with the fully-local invariant. pulldown-cmark = "0.12" printpdf = "0.7" -zip = { version = "2", default-features = false, features = ["deflate"] } # unzip the on-demand NPU runtime bundle (T3.4) docx-rs = "0.4" +# Extracts the on-demand NPU/DirectML runtime bundles (7z, LZMA2+BCJ). Optional + +# npu-gated so the CPU-only build stays lean (NFR-MNT-4). Pinned to 0.7.0: newer +# releases raise the MSRV above the project's rust-version. +sevenz-rust2 = { version = "0.7.0", optional = true } [target.'cfg(windows)'.dependencies] windows = { version = "0.58", features = [ @@ -93,7 +96,7 @@ cpu-transcription = ["dep:whisper-rs"] # whisper-rs CPU build # Phase 3 acceleration (built on capable CI runners; never required) cuda = ["whisper-rs?/cuda"] # whisper.cpp CUDA vulkan = ["whisper-rs?/vulkan"] # whisper.cpp Vulkan -npu = ["dep:ort", "dep:rustfft"] # ort + OpenVINO EP NPU path (T3.4) +npu = ["dep:ort", "dep:rustfft", "dep:sevenz-rust2"] # ort + OpenVINO EP NPU path (T3.4); 7z for runtime bundles # Phase 4 / 6 (added when integrated) diarization = ["dep:sherpa-rs"] # sherpa-onnx pst = [] # shells out to readpst (libpst) — no crate dep, see ADR-0008 update diff --git a/src-tauri/build.rs b/src-tauri/build.rs index 261851f..4784e65 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -1,3 +1,18 @@ fn main() { + // Bake the short commit hash in for the About page. Falls back to "unknown" + // in a non-git build (e.g. a source tarball). `logs/HEAD` is the reflog — it + // gets a line on every commit/checkout, so watching it re-runs this script + // when the hash changes (build.rs output is otherwise cached). + let hash = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "unknown".to_string()); + println!("cargo:rustc-env=WA_GIT_HASH={hash}"); + println!("cargo:rerun-if-changed=../.git/logs/HEAD"); + tauri_build::build(); } diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 64e0373..93243e8 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -119,25 +119,44 @@ fn load_transcriber( backend: BackendId, whisper_model: &Path, ) -> Result<(Box, BackendId), crate::transcription::TrxError> { + use crate::hardware::{resolve_accel, AccelPath}; + + // Resolve how this backend is actually served in *this* build (CUDA/Vulkan + // baked in? NPU/DirectML runtime present?) rather than assuming a GPU + // backend has a working accel path just because the hardware exists. + let path = resolve_accel(backend); + + // ONNX engine — NPU (OpenVINO EP) or AMD/Intel GPU (DirectML EP). Same model + // artifacts either way; the EP is chosen inside OnnxTranscriber::load from + // `backend`, and the resolved `backend` is reported so the UI shows the real + // engine. #[cfg(feature = "npu")] - if backend == BackendId::Npu { - use crate::transcription::{onnx_models, OnnxNpuTranscriber}; + if matches!(path, AccelPath::OnnxOpenVino | AccelPath::OnnxDirectML) { + use crate::transcription::{onnx_models, OnnxTranscriber}; if onnx_models::is_installed(onnx_models::DEFAULT_ONNX_MODEL) { let dir = onnx_models::model_dir(onnx_models::DEFAULT_ONNX_MODEL); - match OnnxNpuTranscriber::load(&dir, BackendId::Npu) { - Ok(t) => return Ok((Box::new(t), BackendId::Npu)), - Err(e) => tracing::warn!("NPU engine load failed ({e}); falling back to CPU"), + match OnnxTranscriber::load(&dir, backend) { + Ok(t) => return Ok((Box::new(t), backend)), + Err(e) => tracing::warn!("ONNX engine load failed ({e}); falling back to CPU"), } } else { tracing::warn!( - "NPU selected but ONNX model not installed; falling back to whisper.cpp" + "ONNX backend selected but model not installed; falling back to whisper.cpp" ); } } - match WhisperTranscriber::load(whisper_model, backend) { - Ok(t) => Ok((Box::new(t), backend)), - Err(e) if backend != BackendId::Cpu => { - tracing::warn!("backend {backend:?} failed to load ({e}); falling back to CPU"); + + // whisper.cpp path: only ask for GPU offload when the resolver picked a + // whisper GPU backend that's compiled in — otherwise a bogus `use_gpu` for a + // vendor with no accel path just no-ops. Anything else decodes on the CPU. + let whisper_backend = match path { + AccelPath::WhisperCuda | AccelPath::WhisperVulkan => backend, + _ => BackendId::Cpu, + }; + match WhisperTranscriber::load(whisper_model, whisper_backend) { + Ok(t) => Ok((Box::new(t), whisper_backend)), + Err(e) if whisper_backend != BackendId::Cpu => { + tracing::warn!("backend {whisper_backend:?} failed to load ({e}); falling back to CPU"); WhisperTranscriber::load(whisper_model, BackendId::Cpu) .map(|t| (Box::new(t) as Box, BackendId::Cpu)) } @@ -815,6 +834,42 @@ pub async fn map_speaker_to_participant( Ok(()) } +// ---- App metadata (About page) ---- + +/// Version + build commit for the About page. Version comes from Cargo; the +/// short commit hash is baked in at build time by `build.rs` (`WA_GIT_HASH`). +#[tauri::command] +pub async fn app_info() -> WaResult { + Ok(serde_json::json!({ + "version": env!("CARGO_PKG_VERSION"), + "commit": env!("WA_GIT_HASH"), + })) +} + +/// Opens an http(s) URL in the user's default browser (About page source link). +/// Windows-only (WA is Windows-native, ADR-0001). The scheme is validated so +/// this can't be coerced into launching a local path or program, and `explorer` +/// receives the URL as a single argv (no shell), so there's no injection surface. +#[tauri::command] +pub async fn open_url(url: String) -> WaResult<()> { + if !(url.starts_with("https://") || url.starts_with("http://")) { + return Err(WaError::new("app", "only http(s) URLs may be opened")); + } + #[cfg(windows)] + { + std::process::Command::new("explorer") + .arg(&url) + .spawn() + .map_err(|e| WaError::new("app", e.to_string()))?; + Ok(()) + } + #[cfg(not(windows))] + { + let _ = url; + Err(WaError::new("app", "unsupported platform")) + } +} + // ---- Hardware + models (Phase 3) ---- #[tauri::command] @@ -837,6 +892,14 @@ pub async fn hardware_status() -> WaResult { "runtimeReady": crate::paths::npu_runtime_ready(), "modelInstalled": npu_model_installed(), }, + // DirectML GPU package (P2): shares the ONNX model with the NPU path; + // only the runtime differs. `applicable` gates the Settings ▸ Hardware + // card so it only shows when DirectML would actually help this build. + "directml": { + "applicable": crate::hardware::directml_would_help(), + "runtimeReady": crate::paths::directml_runtime_ready(), + "modelInstalled": npu_model_installed(), + }, })) } @@ -885,15 +948,27 @@ pub async fn download_npu_model(app: AppHandle) -> WaResult<()> { .map_err(|e| WaError::new("npu", e.to_string())) } -/// Hosted OpenVINO runtime bundle (ORT 1.24.1 + OpenVINO 2025.4.1 DLLs, zipped). -/// Overridable at runtime via `WA_NPU_RUNTIME_URL`. Keep the SHA-256 in step with -/// the uploaded bundle (see `dist/whispassist-npu-runtime-win-x64.zip`). +/// Hosted OpenVINO runtime bundle (ORT 1.24.1 + OpenVINO 2025.4.1 DLLs, 7z). +/// Served from the repo's `runtime/` dir via gitea's `media` path (resolves the +/// Git LFS object, unlike `raw` which returns the pointer); overridable at +/// runtime via `WA_NPU_RUNTIME_URL`. Keep the SHA-256 in step with the file. #[cfg(feature = "npu")] const NPU_RUNTIME_URL: &str = - "https://git.dou.bet/api/packages/iamdoubz/generic/npu-runtime/2025.4.1/whispassist-npu-runtime-win-x64.zip"; + "https://git.dou.bet/iamdoubz/WhispAssist/media/branch/main/runtime/openvino.7z"; /// SHA-256 of the runtime bundle; empty string disables the integrity check. #[cfg(feature = "npu")] -const NPU_RUNTIME_SHA256: &str = "c60de07b5b1ddc2fd1e966d8275d9f55ec261efc81355ea814d6dac897adbdc5"; +const NPU_RUNTIME_SHA256: &str = "ca0be9fc52c78ee623b152f790450b3d4020c5a7ebe99d27736455b308782191"; + +/// Hosted DirectML runtime bundle (ORT 1.24.1 DirectML-EP DLLs, 7z), same repo +/// `media` (LFS-resolving) path as the OpenVINO one; overridable via +/// `WA_DIRECTML_RUNTIME_URL`. +#[cfg(feature = "npu")] +const DIRECTML_RUNTIME_URL: &str = + "https://git.dou.bet/iamdoubz/WhispAssist/media/branch/main/runtime/directml.7z"; +/// SHA-256 of the DirectML runtime bundle; empty string disables the check. +#[cfg(feature = "npu")] +const DIRECTML_RUNTIME_SHA256: &str = + "34369222fcc1be2e72a957b868b1976a90150ba704a06c9e8992c34ee368926b"; /// Stages the ONNX Runtime + OpenVINO DLLs into the app's NPU runtime dir by /// downloading the hosted bundle and unzipping it (T3.4). `WA_NPU_RUNTIME_SRC` @@ -913,7 +988,36 @@ async fn stage_npu_runtime(app: &AppHandle) -> WaResult<()> { } let url = std::env::var("WA_NPU_RUNTIME_URL").unwrap_or_else(|_| NPU_RUNTIME_URL.to_string()); - download_and_extract_runtime(app, &dir, &url).await + download_and_extract_runtime( + app, + &dir, + &url, + NPU_RUNTIME_SHA256, + crate::paths::npu_runtime_ready, + ) + .await +} + +/// Stages the DirectML ONNX Runtime into the app's DirectML runtime dir (P2) by +/// downloading the hosted 7z bundle and unpacking it. Overridable via +/// `WA_DIRECTML_RUNTIME_URL`; dev testing can also just set `ORT_DYLIB_PATH`. +#[cfg(feature = "npu")] +async fn stage_directml_runtime(app: &AppHandle) -> WaResult<()> { + if crate::paths::directml_runtime_ready() { + return Ok(()); + } + let dir = crate::paths::directml_runtime_dir(); + std::fs::create_dir_all(&dir).map_err(|e| WaError::new("directml", e.to_string()))?; + let url = std::env::var("WA_DIRECTML_RUNTIME_URL") + .unwrap_or_else(|_| DIRECTML_RUNTIME_URL.to_string()); + download_and_extract_runtime( + app, + &dir, + &url, + DIRECTML_RUNTIME_SHA256, + crate::paths::directml_runtime_ready, + ) + .await } #[cfg(feature = "npu")] @@ -951,10 +1055,21 @@ fn stage_npu_runtime_from_local( Ok(()) } -/// Downloads the runtime bundle (streaming progress + SHA-256 check) and unzips -/// its DLLs flat into `dir`. +/// Downloads a runtime bundle (streaming progress + SHA-256 check) and unzips +/// its DLLs flat into `dir`. `sha256` empty disables the integrity check; +/// `ready` is the on-disk readiness predicate for the target runtime (NPU or +/// DirectML), so this one function serves both. +// ponytail: progress event + error domain stay "npu"/"npu://download" even on +// the DirectML path — cosmetic only, no frontend consumer for a directml +// channel yet. Split them out when the Settings UI grows a DirectML indicator. #[cfg(feature = "npu")] -async fn download_and_extract_runtime(app: &AppHandle, dir: &Path, url: &str) -> WaResult<()> { +async fn download_and_extract_runtime( + app: &AppHandle, + dir: &Path, + url: &str, + sha256: &str, + ready: fn() -> bool, +) -> WaResult<()> { use futures_util::StreamExt; use sha2::{Digest, Sha256}; @@ -968,7 +1083,7 @@ async fn download_and_extract_runtime(app: &AppHandle, dir: &Path, url: &str) -> )); } let total = resp.content_length(); - let tmp = dir.join("runtime.zip.part"); + let tmp = dir.join("runtime.7z.part"); let mut file = std::fs::File::create(&tmp).map_err(|e| WaError::new("npu", e.to_string()))?; let mut hasher = Sha256::new(); let mut received = 0u64; @@ -987,21 +1102,21 @@ async fn download_and_extract_runtime(app: &AppHandle, dir: &Path, url: &str) -> drop(file); let digest = format!("{:x}", hasher.finalize()); - if !NPU_RUNTIME_SHA256.is_empty() && digest != NPU_RUNTIME_SHA256 { + if !sha256.is_empty() && digest != sha256 { let _ = std::fs::remove_file(&tmp); return Err(WaError::new("npu", "runtime bundle checksum mismatch")); } - // Unzip off the async runtime (CPU/IO-bound). + // Un-7z off the async runtime (CPU/IO-bound). let tmp_for_unzip = tmp.clone(); let dir_for_unzip = dir.to_path_buf(); - tokio::task::spawn_blocking(move || extract_zip_flat(&tmp_for_unzip, &dir_for_unzip)) + tokio::task::spawn_blocking(move || extract_7z_flat(&tmp_for_unzip, &dir_for_unzip)) .await .map_err(|e| WaError::new("npu", e.to_string()))? .map_err(|e| WaError::new("npu", e))?; let _ = std::fs::remove_file(&tmp); - if !crate::paths::npu_runtime_ready() { + if !ready() { return Err(WaError::new( "npu", "runtime bundle extracted but onnxruntime.dll is missing", @@ -1011,27 +1126,23 @@ async fn download_and_extract_runtime(app: &AppHandle, dir: &Path, url: &str) -> } /// Extract every file entry of a zip into `dir`, flattening paths to just the -/// file name (which also prevents zip-slip path traversal). +/// file name (which also strips the archive's top folder — the bundles nest DLLs +/// under `directml/` or `openvino/` — and prevents path traversal). #[cfg(feature = "npu")] -fn extract_zip_flat(zip_path: &Path, dir: &Path) -> Result<(), String> { - let file = std::fs::File::open(zip_path).map_err(|e| e.to_string())?; - let mut archive = zip::ZipArchive::new(file).map_err(|e| e.to_string())?; - for i in 0..archive.len() { - let mut entry = archive.by_index(i).map_err(|e| e.to_string())?; - if entry.is_dir() { - continue; +fn extract_7z_flat(archive_path: &Path, dir: &Path) -> Result<(), String> { + sevenz_rust2::decompress_file_with_extract_fn(archive_path, dir, |entry, reader, _dest| { + if entry.is_directory() { + return Ok(true); } - let Some(name) = Path::new(entry.name()) - .file_name() - .and_then(|n| n.to_str()) - .map(str::to_string) - else { - continue; + // Flatten to just the file name, dropping any folder prefix. + let Some(name) = Path::new(entry.name()).file_name().and_then(|n| n.to_str()) else { + return Ok(true); }; - let mut out = std::fs::File::create(dir.join(name)).map_err(|e| e.to_string())?; - std::io::copy(&mut entry, &mut out).map_err(|e| e.to_string())?; - } - Ok(()) + let mut out = std::fs::File::create(dir.join(name)).map_err(sevenz_rust2::Error::io)?; + std::io::copy(reader, &mut out).map_err(sevenz_rust2::Error::io)?; + Ok(true) + }) + .map_err(|e| e.to_string()) } /// Downloads everything the NPU engine needs (ONNX model + OpenVINO runtime) for @@ -1055,6 +1166,31 @@ pub async fn download_npu_package(app: AppHandle) -> WaResult<()> { } } +/// Downloads everything the DirectML GPU engine needs — the same Whisper ONNX +/// model as the NPU path, plus the DirectML runtime (P2). The non-Vulkan GPU +/// path for AMD/Intel. +#[tauri::command] +pub async fn download_directml_package(app: AppHandle) -> WaResult<()> { + #[cfg(feature = "npu")] + { + download_npu_model(app.clone()).await?; // identical ONNX artifacts + stage_directml_runtime(&app).await?; + let _ = app.emit( + "directml://download", + serde_json::json!({ "stage": "done", "ready": crate::paths::directml_runtime_ready() }), + ); + Ok(()) + } + #[cfg(not(feature = "npu"))] + { + let _ = app; + Err(WaError::new( + "directml", + "this build has no ONNX/DirectML support", + )) + } +} + #[tauri::command] pub async fn list_models() -> WaResult> { let settings = load_settings(); @@ -2732,6 +2868,35 @@ pub async fn privacy_self_check(state: State<'_, AppState>) -> WaResult TranscriptSegment { TranscriptSegment { id: 0, diff --git a/src-tauri/src/hardware/mod.rs b/src-tauri/src/hardware/mod.rs index f6965fa..4d23df8 100644 --- a/src-tauri/src/hardware/mod.rs +++ b/src-tauri/src/hardware/mod.rs @@ -29,6 +29,124 @@ fn rank_of(id: BackendId) -> u8 { } } +/// The concrete engine + device an accelerated transcriber will actually use. +/// +/// whisper.cpp bakes in exactly **one** GPU backend at compile time (CUDA *or* +/// Vulkan, never both), so a `BackendId` alone can't say how a given vendor is +/// served — that depends on which Cargo features this binary was built with and +/// which runtimes are present on disk. This resolver is the single source of +/// truth for that, and it's what makes `available` honest: a GPU vendor is only +/// "available" when it resolves to a non-CPU path (before this, DXGI marked an +/// AMD/NVIDIA GPU available even in a build with no accel for it, so `best()` +/// would route to a GPU that silently no-ops back to CPU). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccelPath { + /// whisper.cpp on CPU — the guaranteed fallback for any vendor with no + /// compiled accel path in this build. + WhisperCpu, + /// whisper.cpp offloading to the GPU via its compiled-in Vulkan backend + /// (universal build: NVIDIA/AMD/Intel). + WhisperVulkan, + /// whisper.cpp offloading to the GPU via its compiled-in CUDA backend + /// (NVIDIA-turbo build). + WhisperCuda, + /// Whisper-ONNX on the Intel NPU via ONNX Runtime + OpenVINO EP. + OnnxOpenVino, + /// Whisper-ONNX on a DX12 GPU via ONNX Runtime + DirectML EP — the + /// non-Vulkan GPU path for AMD/Intel (P2). Requires the DirectML-enabled + /// runtime to be present. + OnnxDirectML, +} + +/// Pure resolution logic, factored out of [`resolve_accel`] so every +/// (vendor × compiled-feature × runtime-present) combination is unit-testable +/// without real hardware or a specific feature build. +fn resolve_accel_with( + backend: BackendId, + cuda: bool, + vulkan: bool, + npu_ready: bool, + directml_ready: bool, +) -> AccelPath { + match backend { + BackendId::Cpu => AccelPath::WhisperCpu, + BackendId::Npu => { + if npu_ready { + AccelPath::OnnxOpenVino + } else { + AccelPath::WhisperCpu + } + } + // NVIDIA prefers its native CUDA fast path, then the universal Vulkan + // build, then DirectML, then CPU. + BackendId::Nvidia => { + if cuda { + AccelPath::WhisperCuda + } else if vulkan { + AccelPath::WhisperVulkan + } else if directml_ready { + AccelPath::OnnxDirectML + } else { + AccelPath::WhisperCpu + } + } + // AMD/Intel: Vulkan in the universal build, else DirectML (the + // non-Vulkan path, e.g. inside the CUDA-only turbo variant), else CPU. + BackendId::Amd | BackendId::Intel => { + if vulkan { + AccelPath::WhisperVulkan + } else if directml_ready { + AccelPath::OnnxDirectML + } else { + AccelPath::WhisperCpu + } + } + } +} + +/// Resolves how `backend` is served in *this* binary, reading the compile-time +/// GPU features and on-disk runtime readiness. See [`AccelPath`]. +pub fn resolve_accel(backend: BackendId) -> AccelPath { + resolve_accel_with( + backend, + cfg!(feature = "cuda"), + cfg!(feature = "vulkan"), + cfg!(feature = "npu") && crate::paths::npu_runtime_ready(), + directml_ready(), + ) +} + +/// Whether the DirectML-enabled ONNX engine is usable in this binary: the `ort` +/// engine must be compiled in (`npu` feature) and its DirectML runtime staged. +fn directml_ready() -> bool { + cfg!(feature = "npu") && crate::paths::directml_runtime_ready() +} + +/// True when staging the DirectML runtime would light up a physically-present +/// GPU this build can't otherwise accelerate: the ONNX engine is compiled, no +/// Vulkan backend is baked in (which would already cover every GPU), and a GPU +/// that would then route to DirectML is present — AMD/Intel always, NVIDIA only +/// when there's no CUDA fast path. Drives whether Settings offers the package. +pub fn directml_would_help() -> bool { + if !cfg!(feature = "npu") || cfg!(feature = "vulkan") { + return false; + } + #[cfg(windows)] + { + // Presence, not availability — `available` is false pre-staging, but the + // enumerated entries still tell us which GPUs physically exist. + dxgi::enumerate_gpus().iter().any(|b| match b.id { + BackendId::Amd | BackendId::Intel => true, + BackendId::Nvidia => !cfg!(feature = "cuda"), + _ => false, + }) + } + #[cfg(not(windows))] + { + false + } +} + fn cpu_backend() -> BackendInfo { BackendInfo { id: BackendId::Cpu, @@ -58,10 +176,12 @@ pub fn npu_hardware_present() -> bool { /// download kept out of the base installer (see `paths::npu_runtime_ready`). /// `available` gates routing, so it only flips true once BOTH the chip and its /// runtime are in place — which, by build order (T3.4 step 5 → 1-3), is also -/// after the `OnnxNpuTranscriber` path exists. +/// after the `OnnxTranscriber` path exists. fn npu_backend() -> BackendInfo { let present = npu_hardware_present(); - let ready = present && crate::paths::npu_runtime_ready(); + // Honest availability: the chip must be present AND resolve to a real accel + // path (OpenVINO), which folds in the `npu` feature gate + runtime download. + let available = present && !matches!(resolve_accel(BackendId::Npu), AccelPath::WhisperCpu); BackendInfo { id: BackendId::Npu, name: if present { @@ -69,7 +189,7 @@ fn npu_backend() -> BackendInfo { } else { "NPU".to_string() }, - available: ready, + available, rank: rank_of(BackendId::Npu), vram_mb: None, } @@ -193,6 +313,67 @@ mod tests { let backends = vec![backend(BackendId::Npu, false)]; assert_eq!(pick_best(&backends, None).id, BackendId::Cpu); } + + // --- AccelPath resolution (P0): pure, feature-independent ----------------- + + #[test] + fn cpu_and_npu_ignore_gpu_features() { + // CPU is always CPU. + assert_eq!( + resolve_accel_with(BackendId::Cpu, true, true, true, true), + AccelPath::WhisperCpu + ); + // NPU keys only off its runtime being ready, not any GPU feature. + assert_eq!( + resolve_accel_with(BackendId::Npu, false, false, true, false), + AccelPath::OnnxOpenVino + ); + assert_eq!( + resolve_accel_with(BackendId::Npu, true, true, false, true), + AccelPath::WhisperCpu + ); + } + + #[test] + fn nvidia_prefers_cuda_then_vulkan_then_directml_then_cpu() { + let n = BackendId::Nvidia; + assert_eq!( + resolve_accel_with(n, true, true, false, true), + AccelPath::WhisperCuda // CUDA wins even if Vulkan/DirectML also present + ); + assert_eq!( + resolve_accel_with(n, false, true, false, true), + AccelPath::WhisperVulkan + ); + assert_eq!( + resolve_accel_with(n, false, false, false, true), + AccelPath::OnnxDirectML + ); + assert_eq!( + resolve_accel_with(n, false, false, false, false), + AccelPath::WhisperCpu + ); + } + + #[test] + fn amd_and_intel_take_vulkan_then_directml_then_cpu_and_never_cuda() { + for v in [BackendId::Amd, BackendId::Intel] { + // CUDA is NVIDIA-only: an AMD/Intel GPU in a CUDA-only build must NOT + // claim CUDA — it falls to DirectML (the non-Vulkan path) or CPU. + assert_eq!( + resolve_accel_with(v, true, false, false, true), + AccelPath::OnnxDirectML + ); + assert_eq!( + resolve_accel_with(v, true, false, false, false), + AccelPath::WhisperCpu + ); + assert_eq!( + resolve_accel_with(v, false, true, false, false), + AccelPath::WhisperVulkan + ); + } + } } #[cfg(windows)] @@ -317,10 +498,14 @@ mod dxgi { let name = String::from_utf16_lossy(&desc.Description) .trim_end_matches('\0') .to_string(); + // The adapter is physically present, but it's only *available* for + // routing if this build actually has an accel path for it — else + // `best()` would pick a GPU that no-ops back to CPU. + let available = !matches!(super::resolve_accel(id), super::AccelPath::WhisperCpu); out.push(BackendInfo { id, name, - available: true, + available, rank: rank_of(id), vram_mb: Some((desc.DedicatedVideoMemory / (1024 * 1024)) as u32), }); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 6d39655..b5c334f 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -172,10 +172,13 @@ pub fn run() { commands::set_recording_retention, commands::acknowledge_recording_consent, commands::resume_transcription, + commands::app_info, + commands::open_url, commands::hardware_status, commands::set_preferred_backend, commands::list_models, commands::download_npu_package, + commands::download_directml_package, commands::list_diarization_models, commands::download_model, commands::remove_model, diff --git a/src-tauri/src/paths.rs b/src-tauri/src/paths.rs index 989369e..e4449fe 100644 --- a/src-tauri/src/paths.rs +++ b/src-tauri/src/paths.rs @@ -63,6 +63,26 @@ pub fn npu_runtime_ready() -> bool { npu_runtime_dll().exists() } +/// On-demand DirectML runtime (ONNX Runtime built with the DirectML EP, P2). +/// The non-Vulkan GPU path for AMD/Intel reuses the same Whisper-ONNX model as +/// the NPU path but needs a *different* `onnxruntime.dll` — Intel's OpenVINO ORT +/// build doesn't carry the DirectML EP — so it lives in its own runtime dir. +/// `DirectML.dll` itself ships with Windows 10 1903+, so the bundle is just ORT. +pub fn directml_runtime_dir() -> PathBuf { + wa_root().join("runtime").join("directml") +} + +/// The dlopen target for `ort` (load-dynamic) on the DirectML path; its presence +/// is the readiness signal that gates AMD/Intel non-Vulkan availability. +pub fn directml_runtime_dll() -> PathBuf { + directml_runtime_dir().join("onnxruntime.dll") +} + +/// True once the DirectML runtime has been staged (P2). +pub fn directml_runtime_ready() -> bool { + directml_runtime_dll().exists() +} + /// Fixed filenames pending T4.7 (diarization model management/selection in Settings). pub fn diarization_segmentation_model_file() -> PathBuf { models_dir().join("seg-pyannote-3.0.onnx") diff --git a/src-tauri/src/transcription/mel.rs b/src-tauri/src/transcription/mel.rs index a5b9230..2d3d044 100644 --- a/src-tauri/src/transcription/mel.rs +++ b/src-tauri/src/transcription/mel.rs @@ -1,7 +1,7 @@ //! Whisper log-mel spectrogram front-end for the NPU path (T3.4). //! //! whisper.cpp computed the mel internally; ONNX Runtime does not, so the -//! `OnnxNpuTranscriber` needs its own. This reproduces HuggingFace's +//! `OnnxTranscriber` needs its own. This reproduces HuggingFace's //! `WhisperFeatureExtractor` **exactly** (same window/hop/filters/normalization) //! because the ONNX encoder we run was exported against that preprocessing — //! any deviation feeds the encoder out-of-distribution features and garbles the diff --git a/src-tauri/src/transcription/mod.rs b/src-tauri/src/transcription/mod.rs index 0db9145..3de5be6 100644 --- a/src-tauri/src/transcription/mod.rs +++ b/src-tauri/src/transcription/mod.rs @@ -199,9 +199,10 @@ fn audio_ctx_for_window(n_samples: usize) -> i32 { ((window_secs / 30.0) * 1500.0).ceil().clamp(64.0, 1500.0) as i32 } -// NPU tier (T3.4): Whisper ONNX on the Intel NPU via ONNX Runtime + OpenVINO. -// A real `OnnxNpuTranscriber` implementing the same `Transcriber` trait; see -// `npu.rs`. Gated on the `npu` feature so the CPU-only build never pulls `ort`. +// ONNX tier: Whisper ONNX via ONNX Runtime on the Intel NPU (OpenVINO, T3.4) or +// a DX12 GPU (DirectML — the non-Vulkan AMD/Intel path, P2). One +// `OnnxTranscriber` implementing the same `Transcriber` trait; see `npu.rs`. +// Gated on the `npu` feature so the CPU-only build never pulls `ort`. #[cfg(feature = "npu")] pub mod mel; #[cfg(feature = "npu")] @@ -209,7 +210,7 @@ pub mod npu; #[cfg(feature = "npu")] pub mod onnx_models; #[cfg(feature = "npu")] -pub use npu::OnnxNpuTranscriber; +pub use npu::OnnxTranscriber; /// Streaming window worker (Phase 1, T1.5/T1.6): accumulates raw 16kHz-mono /// chunks from the `audio` service into fixed-size, **non-overlapping** windows diff --git a/src-tauri/src/transcription/npu.rs b/src-tauri/src/transcription/npu.rs index 9fde9cb..b8de977 100644 --- a/src-tauri/src/transcription/npu.rs +++ b/src-tauri/src/transcription/npu.rs @@ -1,12 +1,15 @@ -//! NPU transcriber: Whisper ONNX on the Intel NPU via ONNX Runtime + OpenVINO -//! (T3.4, ADR-0004). Same `Transcriber` trait and `TranscriptSegment` output as -//! the whisper.cpp path, so callers never branch on engine. +//! ONNX transcriber: Whisper ONNX via ONNX Runtime, on either the Intel NPU +//! (OpenVINO EP, T3.4) or a DX12 GPU (DirectML EP, P2 — the non-Vulkan path for +//! AMD/Intel). One engine, EP chosen by the requested `BackendId`; same +//! `Transcriber` trait and `TranscriptSegment` output as the whisper.cpp path, +//! so callers never branch on engine. The ONNX model artifacts are identical +//! across EPs (an EP is a device backend, not a different graph). //! //! Split of work, validated by the T3.4 spike (encoder ~3.6× faster on NPU): -//! - **encoder** (fixed `[1,80,3000]` shape) runs on the **NPU** via OpenVINO — -//! the expensive graph, and the shape NPUs want; +//! - **encoder** (fixed `[1,80,3000]` shape) runs on the **accelerator** (NPU or +//! GPU) — the expensive graph, and the fixed shape accelerators want; //! - **decoder** (dynamic, autoregressive) runs greedy on the **CPU** EP — a -//! dynamic KV loop is a poor NPU fit and the cheap half anyway. +//! dynamic KV loop is a poor accelerator fit and the cheap half anyway. //! //! ponytail: greedy, no-KV-cache decode (re-feeds the full token prefix each //! step). Correct and simple; windows are short so the token count is small. @@ -14,7 +17,9 @@ use super::{AudioWindow, SegmentSink, Transcriber, TrxError}; use crate::models::{BackendId, TranscriptSegment}; -use ort::execution_providers::OpenVINOExecutionProvider; +use ort::execution_providers::{ + DirectMLExecutionProvider, ExecutionProviderDispatch, OpenVINOExecutionProvider, +}; use ort::session::Session; use ort::value::Tensor; use std::collections::HashMap; @@ -25,7 +30,7 @@ use std::sync::Mutex; /// Safety cap on generated tokens per 30 s window (Whisper's own max is 448). const MAX_NEW_TOKENS: usize = 224; -pub struct OnnxNpuTranscriber { +pub struct OnnxTranscriber { encoder: Mutex, decoder: Mutex, enc_input: String, @@ -44,15 +49,21 @@ pub struct OnnxNpuTranscriber { next_id: AtomicU64, } -impl OnnxNpuTranscriber { - /// Points `ort` (load-dynamic) at the on-demand-downloaded runtime and puts - /// its DLLs on the search path, unless the caller already set `ORT_DYLIB_PATH` - /// (the test harness does, to target a dev runtime). Idempotent. - fn ensure_runtime_env() { +impl OnnxTranscriber { + /// Points `ort` (load-dynamic) at the given on-demand-downloaded runtime DLL + /// and puts its sibling DLLs on the search path, unless the caller already + /// set `ORT_DYLIB_PATH` (the test harness does, to target a dev runtime). + /// Idempotent. + /// + /// `ORT_DYLIB_PATH` is process-global and `ort` dlopens it once, so a single + /// process run must not mix EPs from different runtime bundles (OpenVINO vs + /// DirectML). That's fine: `best()` picks one backend per session. + // ponytail: one runtime per process. If backend hot-swapping between NPU and + // DirectML in one run is ever needed, that's a re-init-ort problem to solve then. + fn ensure_runtime_env(dll: std::path::PathBuf) { if std::env::var_os("ORT_DYLIB_PATH").is_some() { return; } - let dll = crate::paths::npu_runtime_dll(); std::env::set_var("ORT_DYLIB_PATH", &dll); if let Some(dir) = dll.parent() { let path = std::env::var_os("PATH").unwrap_or_default(); @@ -185,32 +196,52 @@ impl OnnxNpuTranscriber { } } -impl Transcriber for OnnxNpuTranscriber { - /// `model` is the directory holding the ONNX artifacts (see `onnx_models`). - /// `backend` is expected to be `Npu`; a non-NPU value is rejected so the - /// dispatcher can fall back to whisper.cpp rather than us guessing. +impl Transcriber for OnnxTranscriber { + /// `model` is the directory holding the ONNX artifacts (see `onnx_models`) — + /// the same artifacts for every EP. `backend` selects the accelerator: + /// `Npu` → OpenVINO EP; `Amd`/`Intel` → DirectML EP (the non-Vulkan GPU + /// path). Any other value is rejected so the dispatcher can fall back to + /// whisper.cpp rather than us guessing. fn load(model: &Path, backend: BackendId) -> Result { - if backend != BackendId::Npu { - return Err(TrxError::Load(format!( - "OnnxNpuTranscriber only serves the NPU backend, got {backend:?}" - ))); - } - Self::ensure_runtime_env(); + // Pick the runtime bundle + encoder EP for the requested accelerator. + // error_on_failure makes a failed accelerator registration LOUD (Err) + // instead of a silent CPU fallback, so the dispatcher can cleanly drop + // to the whisper.cpp path (T3.4 spike lesson). + let (runtime_dll, encoder_ep): (std::path::PathBuf, ExecutionProviderDispatch) = + match backend { + BackendId::Npu => ( + crate::paths::npu_runtime_dll(), + OpenVINOExecutionProvider::default() + .with_device_type("NPU") + .build() + .error_on_failure(), + ), + // ponytail: DirectML device_id 0 = the default DX12 adapter. + // Map the DXGI adapter index here if a multi-GPU box ever needs + // to pin a specific AMD/Intel card. + BackendId::Amd | BackendId::Intel => ( + crate::paths::directml_runtime_dll(), + DirectMLExecutionProvider::default() + .with_device_id(0) + .build() + .error_on_failure(), + ), + other => { + return Err(TrxError::Load(format!( + "OnnxTranscriber serves NPU/AMD/Intel, got {other:?}" + ))) + } + }; + Self::ensure_runtime_env(runtime_dll); let encoder_path = model.join("encoder_model.onnx"); let decoder_path = model.join("decoder_model.onnx"); let tokenizer_path = model.join("tokenizer.json"); let gen_config_path = model.join("generation_config.json"); - // Encoder pinned to the NPU. error_on_failure makes a failed NPU - // registration LOUD (Err) instead of a silent CPU fallback, so the - // dispatcher can cleanly drop to the whisper.cpp path (T3.4 spike lesson). let encoder = Session::builder() .map_err(|e| TrxError::Load(e.to_string()))? - .with_execution_providers([OpenVINOExecutionProvider::default() - .with_device_type("NPU") - .build() - .error_on_failure()]) + .with_execution_providers([encoder_ep]) .map_err(|e| TrxError::Load(e.to_string()))? .commit_from_file(&encoder_path) .map_err(|e| TrxError::Load(e.to_string()))?; @@ -434,7 +465,7 @@ mod tests { }; let wav = std::env::var("WA_NPU_TEST_WAV").expect("set WA_NPU_TEST_WAV"); let t0 = std::time::Instant::now(); - let t = OnnxNpuTranscriber::load(Path::new(&model_dir), BackendId::Npu) + let t = OnnxTranscriber::load(Path::new(&model_dir), BackendId::Npu) .expect("load NPU transcriber"); let load_ms = t0.elapsed().as_millis(); let t1 = std::time::Instant::now(); @@ -455,4 +486,45 @@ mod tests { ); } } + + /// Real DirectML GPU inference (P2), opt-in. Point `ORT_DYLIB_PATH` at a + /// DirectML-EP `onnxruntime.dll` (or stage `runtime\directml\`), reuse the + /// same ONNX model dir as the NPU spike, and run on the Intel/AMD GPU: + /// ORT_DYLIB_PATH=…directml\onnxruntime.dll WA_DML_MODEL_DIR=… \ + /// WA_DML_TEST_WAV=… cargo test --release --features npu \ + /// directml_transcribes -- --ignored --nocapture + #[test] + #[ignore = "requires a DirectML runtime + model + wav; run manually on a GPU"] + fn directml_transcribes_speech() { + let Some(model_dir) = std::env::var_os("WA_DML_MODEL_DIR") else { + eprintln!("skip: set WA_DML_MODEL_DIR"); + return; + }; + let wav = std::env::var("WA_DML_TEST_WAV").expect("set WA_DML_TEST_WAV"); + // Intel and AMD both take the DirectML EP; either BackendId exercises it. + let backend = match std::env::var("WA_DML_BACKEND").as_deref() { + Ok("amd") => BackendId::Amd, + _ => BackendId::Intel, + }; + let t0 = std::time::Instant::now(); + let t = OnnxTranscriber::load(Path::new(&model_dir), backend).expect("load DirectML"); + let load_ms = t0.elapsed().as_millis(); + let t1 = std::time::Instant::now(); + let segs = t.transcribe_file(Path::new(&wav)).expect("transcribe"); + let infer_ms = t1.elapsed().as_millis(); + let text = segs + .iter() + .map(|s| s.text.as_str()) + .collect::>() + .join(" ") + .to_lowercase(); + eprintln!("[spike] backend={backend:?} load={load_ms}ms infer={infer_ms}ms text={text:?}"); + assert!(!text.trim().is_empty(), "transcript was empty"); + if let Ok(expect) = std::env::var("WA_DML_EXPECT") { + assert!( + text.contains(&expect.to_lowercase()), + "transcript {text:?} missing expected {expect:?}" + ); + } + } } diff --git a/src-tauri/src/transcription/onnx_models.rs b/src-tauri/src/transcription/onnx_models.rs index 437f7ef..083d935 100644 --- a/src-tauri/src/transcription/onnx_models.rs +++ b/src-tauri/src/transcription/onnx_models.rs @@ -14,7 +14,7 @@ use std::path::PathBuf; const REPO: &str = "onnx-community/whisper-base.en"; pub const DEFAULT_ONNX_MODEL: &str = "base.en"; -/// The four artifacts the `OnnxNpuTranscriber` needs, relative to the repo root. +/// The four artifacts the `OnnxTranscriber` needs, relative to the repo root. const FILES: &[(&str, &str)] = &[ ("encoder_model.onnx", "onnx/encoder_model.onnx"), ("decoder_model.onnx", "onnx/decoder_model.onnx"), @@ -34,7 +34,7 @@ fn onnx_dir(id: &str) -> PathBuf { models_dir().join("onnx-whisper").join(id) } -/// Directory holding a model's ONNX artifacts — what `OnnxNpuTranscriber::load` +/// Directory holding a model's ONNX artifacts — what `OnnxTranscriber::load` /// expects as its `model` path. pub fn model_dir(id: &str) -> PathBuf { onnx_dir(id) diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 3e7487e..c99af92 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "WhispAssist", - "version": "0.1.4", + "version": "0.1.5", "identifier": "bet.dou.whispassist", "build": { "frontendDist": "../dist", diff --git a/src/App.svelte b/src/App.svelte index 84ae6b7..2684b58 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -301,6 +301,10 @@ contrast holds in both; every foreground/background pairing below is checked to >=4.5:1 (WCAG AA) against the surface it's meant to sit on. */ :global(:root) { + /* Render native controls (select option popups, textareas, checkboxes, + scrollbars) in the active theme — without this they default to light, + showing bright-white dropdowns/inputs in dark mode. */ + color-scheme: light; --bg: #ffffff; --bg-subtle: #f7f8f9; --bg-elevated: #ffffff; @@ -329,6 +333,7 @@ --radius-full: 999px; } :global([data-theme="dark"]) { + color-scheme: dark; --bg: #14161a; --bg-subtle: #191c22; --bg-elevated: #1c1f26; @@ -486,7 +491,10 @@ } } .theme-select { - background: transparent; + /* Explicit surface (not `transparent`) so the native options popup inherits + a themed background instead of defaulting to white — matches the elevated + header it sits on, so the closed control looks unchanged. */ + background: var(--bg-elevated); border: 1px solid var(--border); color: var(--fg); border-radius: var(--radius-sm); @@ -494,6 +502,12 @@ font-size: 0.8rem; cursor: pointer; } + /* Author the option rows too — Chromium renders an author-styled select's + popup in light unless the options carry their own themed colors. */ + .theme-select option { + background: var(--bg-elevated); + color: var(--fg); + } .backend { font-size: 0.7rem; text-transform: uppercase; diff --git a/src/lib/api.ts b/src/lib/api.ts index bbea864..f530ea5 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -32,6 +32,11 @@ export interface BackendInfo { vram_mb: number | null; } +export interface AppInfo { + version: string; + commit: string; +} + export interface HardwareStatus { backends: BackendInfo[]; active: BackendId; @@ -39,6 +44,8 @@ export interface HardwareStatus { estRtf: number; /** NPU package state (T3.4): chip detected, runtime staged, model fetched. */ npu?: { present: boolean; runtimeReady: boolean; modelInstalled: boolean }; + /** DirectML GPU package (P2): applicable = staging would help this build. */ + directml?: { applicable: boolean; runtimeReady: boolean; modelInstalled: boolean }; } export interface LlmStatus { @@ -318,10 +325,13 @@ export const api = { invoke("set_recording_retention", { meetingId, record }), acknowledgeRecordingConsent: () => invoke("acknowledge_recording_consent"), + appInfo: () => invoke("app_info"), + openUrl: (url: string) => invoke("open_url", { url }), hardwareStatus: () => invoke("hardware_status"), setPreferredBackend: (backend: BackendId | "auto") => invoke("set_preferred_backend", { args: { backend } }), downloadNpuPackage: () => invoke("download_npu_package"), + downloadDirectmlPackage: () => invoke("download_directml_package"), listModels: () => invoke("list_models"), downloadModel: (id: string, kind: "whisper" = "whisper") => invoke("download_model", { args: { kind, id } }), diff --git a/src/lib/views/Settings.svelte b/src/lib/views/Settings.svelte index 5bcddb2..73d29f4 100644 --- a/src/lib/views/Settings.svelte +++ b/src/lib/views/Settings.svelte @@ -24,6 +24,7 @@ RefreshCw, ChevronRight, RotateCcw, + Info, } from "@lucide/svelte"; import { OLLAMA_OPTIONS, @@ -36,9 +37,13 @@ let { onClose }: { onClose: () => void } = $props(); let section = $state< - "recording" | "hardware" | "storage" | "calendar" | "sync" | "ai" | "privacy" + "recording" | "hardware" | "storage" | "calendar" | "sync" | "ai" | "privacy" | "about" >("recording"); + // ---- About (version + build commit + source) ---- + const SOURCE_URL = "https://git.dou.bet/iamdoubz/WhispAssist"; + let appInfo = $state<{ version: string; commit: string } | null>(null); + // ---- AI summary provider (T5.2, FR-LLM-1) ---- let llmProvider = $state(settings.settings.llm_provider); let llmEndpoint = $state(settings.settings.llm_endpoint); @@ -269,9 +274,32 @@ } } + // ---- DirectML package download (P2) ---- + // ponytail: the await resolves when model+runtime are both staged, so the + // busy flag is the whole progress story here — no % listener like NPU has. + let dmlBusy = $state(false); + let dmlMsg = $state(null); + async function downloadDirectmlPackage() { + dmlBusy = true; + dmlMsg = "Downloading…"; + try { + await api.downloadDirectmlPackage(); + await settings.loadHardware(); + dmlMsg = "DirectML package ready."; + } catch (e) { + dmlMsg = `Failed: ${errorMessage(e)}`; + } finally { + dmlBusy = false; + } + } + onMount(() => { settings.load(); calendar.load(); + api + .appInfo() + .then((i) => (appInfo = i)) + .catch(() => (appInfo = null)); const un = events.onNpuDownload((p) => { if (p.stage === "model") npuMsg = p.total ? `Model ${Math.round((100 * (p.received ?? 0)) / p.total)}%` : "Model…"; @@ -389,6 +417,9 @@ + + {/if} + {#if dmlMsg}

{dmlMsg}

{/if} + + {/if} {:else}

Hardware detection unavailable.

{/if} @@ -969,7 +1029,7 @@ {/if} - {:else} + {:else if section === "privacy"}

Privacy

@@ -1074,6 +1134,22 @@ {#if vaultMsg}

{vaultMsg}

{/if} {/if}
+ {:else if section === "about"} +
+

About

+

+ WhispAssist v{appInfo?.version ?? "…"} +

+

A fully local, open-source, Windows-native meeting assistant.

+
+ Build commit{appInfo?.commit ?? "…"} +
+

+ +

+
{/if} @@ -1103,13 +1179,19 @@ top: 0; background: var(--bg-elevated); display: flex; - align-items: center; + align-items: flex-start; gap: 1rem; padding: 0.9rem 0; border-bottom: 1px solid var(--border); } + /* nav takes the middle and wraps its tabs onto a second row instead of + overflowing the fixed-width panel — that overflow was pushing the close + button off the panel edge and adding a horizontal scrollbar. */ nav { display: flex; + flex: 1 1 auto; + min-width: 0; + flex-wrap: wrap; gap: 0.25rem; } nav button, @@ -1139,6 +1221,7 @@ .close { display: grid; place-items: center; + flex: 0 0 auto; margin-left: auto; width: 30px; height: 30px; @@ -1236,7 +1319,8 @@ grid-column: 1 / -1; } input, - select { + select, + textarea { background: var(--bg); color: var(--fg); border: 1px solid var(--border); @@ -1369,6 +1453,23 @@ .link.danger { color: var(--danger); } + .about-name { + font-size: 1.05rem; + font-weight: 600; + margin: 0.5rem 0 0.25rem; + } + .about-ver { + color: var(--muted); + font-weight: 500; + } + /* The source link reads as a link (accent + underline), unlike the muted + .link buttons used for secondary actions elsewhere. */ + .link.source { + color: var(--accent); + text-decoration: underline; + padding: 0; + font: inherit; + } .test { display: inline-flex; align-items: center;