From 7490ff5db4fd34d9980fe46b59fb5d44e1a38c1e Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Tue, 7 Jul 2026 13:06:00 -0500 Subject: [PATCH] feat(transcription): wire language selection through recording/reprocess/recovery (T8.7, M4.2) --- src-tauri/src/commands.rs | 140 +++++++++++++++++++++++++++++++------- 1 file changed, 116 insertions(+), 24 deletions(-) diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index af8a465..f373b74 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -38,6 +38,12 @@ pub struct StartRecordingArgs { pub record: bool, /// Note-template id (Phase 8, T8.1, FR-NOTE-5) — see `notes::built_in_note_templates`. pub template_id: Option, + /// Per-meeting transcription language override (T8.7, FR-TRX-4): `None` + /// falls back to `Settings.whisper_language`; `None`/`"auto"` (either + /// way) requests auto-detection. Only takes effect with a multilingual + /// model — see `transcription::resolve_language`. + #[serde(default)] + pub language: Option, } // ---- File-backed settings (consent/default-retention/storage policy) ---- @@ -54,6 +60,7 @@ fn default_settings() -> Settings { llm_advanced: serde_json::Value::Null, preferred_backend: "auto".into(), whisper_model: crate::paths::DEFAULT_WHISPER_MODEL.to_string(), + whisper_language: None, // auto-detect by default (T8.7, FR-TRX-4) low_overhead: false, default_record: false, consent_acknowledged: false, @@ -129,6 +136,7 @@ fn backend_for(settings: &Settings) -> BackendId { fn load_transcriber( backend: BackendId, whisper_model: &Path, + language: Option<&str>, ) -> Result<(Box, BackendId), crate::transcription::TrxError> { use crate::hardware::{resolve_accel, AccelPath}; @@ -146,7 +154,7 @@ fn load_transcriber( use crate::transcription::{onnx_models, OnnxTranscriber}; if onnx_models::is_installed(onnx_models::DEFAULT_ONNX_MODEL) { let dir = onnx_models::model_dir(onnx_models::DEFAULT_ONNX_MODEL); - match OnnxTranscriber::load(&dir, backend) { + match OnnxTranscriber::load(&dir, backend, language) { Ok(t) => return Ok((Box::new(t), backend)), Err(e) => tracing::warn!("ONNX engine load failed ({e}); falling back to CPU"), } @@ -164,17 +172,29 @@ fn load_transcriber( AccelPath::WhisperCuda | AccelPath::WhisperVulkan => backend, _ => BackendId::Cpu, }; - match WhisperTranscriber::load(whisper_model, whisper_backend) { + match WhisperTranscriber::load(whisper_model, whisper_backend, language) { Ok(t) => Ok((Box::new(t), whisper_backend)), Err(e) if whisper_backend != BackendId::Cpu => { tracing::warn!("backend {whisper_backend:?} failed to load ({e}); falling back to CPU"); - WhisperTranscriber::load(whisper_model, BackendId::Cpu) + WhisperTranscriber::load(whisper_model, BackendId::Cpu, language) .map(|t| (Box::new(t) as Box, BackendId::Cpu)) } Err(e) => Err(e), } } +/// Normalizes a requested language string the same way whisper.cpp itself +/// treats it: an empty string or the literal `"auto"` both mean "no explicit +/// language" — matching `FullParams::set_language`'s own `None`/`Some("auto")` +/// equivalence (T8.7, FR-TRX-4) — so callers can pass either spelling through +/// from settings/args without duplicating this check everywhere. +fn normalize_language(language: Option<&str>) -> Option<&str> { + match language { + Some(l) if !l.is_empty() && !l.eq_ignore_ascii_case("auto") => Some(l), + _ => None, + } +} + fn now_unix() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -276,6 +296,14 @@ pub async fn start_recording( ), )); } + // T8.7/FR-TRX-4: a per-meeting override wins over the Settings default; + // both spellings of "no explicit language" collapse to `None` here. + let language: Option = normalize_language( + args.language + .as_deref() + .or(settings.whisper_language.as_deref()), + ) + .map(|s| s.to_string()); let meeting_id = state .store @@ -285,6 +313,7 @@ pub async fn start_recording( .unwrap_or_else(|| "Untitled meeting".to_string()), calendar_event_id: args.calendar_event_id, template_id: args.template_id, + language: language.clone(), }) .await .map_err(|e| WaError::new("storage", e.to_string()))?; @@ -376,6 +405,14 @@ pub async fn start_recording( let segments_for_worker = segments.clone(); let active_backend: Arc> = Arc::new(StdMutex::new(backend)); let active_backend_for_worker = active_backend.clone(); + // T8.7/FR-TRX-4: starts at the requested value, then the worker refines + // it — first to what the loaded engine actually resolved (e.g. forced + // "en" for an English-only model), then to whatever was last + // used/detected once streaming stops. `stop_recording` reads the final + // value after joining this thread. + let language_state: Arc>> = Arc::new(StdMutex::new(language.clone())); + let language_state_for_worker = language_state.clone(); + let language_for_worker = language.clone(); let app_for_worker = app.clone(); let meeting_id_for_worker = meeting_id.clone(); let transcription_worker = std::thread::Builder::new() @@ -385,13 +422,14 @@ pub async fn start_recording( // fall-through (T3.4/T3.5, FR-HW-4): a GPU/NPU load failure (driver // issue, OOM, missing model) drops to CPU rather than losing the // meeting's transcript entirely. - let (transcriber, used) = match load_transcriber(backend, &model_path) { - Ok(pair) => pair, - Err(e) => { - tracing::error!("failed to load any transcriber: {e}"); - return; - } - }; + let (transcriber, used) = + match load_transcriber(backend, &model_path, language_for_worker.as_deref()) { + Ok(pair) => pair, + Err(e) => { + tracing::error!("failed to load any transcriber: {e}"); + return; + } + }; if used != backend { if let Ok(mut b) = active_backend_for_worker.lock() { *b = used; @@ -404,6 +442,9 @@ pub async fn start_recording( }), ); } + if let Ok(mut lang) = language_state_for_worker.lock() { + *lang = transcriber.effective_language(); + } run_streaming_worker(transcriber.as_ref(), frame_rx, |segment| { if let Ok(mut buf) = segments_for_worker.lock() { buf.push(segment.clone()); @@ -413,6 +454,14 @@ pub async fn start_recording( serde_json::json!({ "meetingId": meeting_id_for_worker, "segment": segment }), ); }); + // Prefer whatever the engine actually used/detected on its last + // decode over the load-time resolution above — meaningful for + // "auto" mode, where the real answer only exists after decoding. + if let Some(detected) = transcriber.detected_language() { + if let Ok(mut lang) = language_state_for_worker.lock() { + *lang = Some(detected); + } + } }) .map_err(|e| WaError::new("transcription", e.to_string()))?; @@ -500,6 +549,7 @@ pub async fn start_recording( segments, active_backend, model_id, + language: language_state, diarizer, speaker_names, }); @@ -586,6 +636,10 @@ pub async fn stop_recording( .map(|b| b.as_str().to_string()) .unwrap_or_else(|_| BackendId::Cpu.as_str().to_string()); + // T8.7/FR-TRX-4: whatever the transcription worker last resolved — + // explicit request, English-only forcing, or auto-detected result. + let language = session.language.lock().ok().and_then(|g| g.clone()); + // T2.10: persist transcript.json (via finalize_meeting) and notes.md // *before* touching the working WAV, so a crash here still leaves a // recoverable, regenerable meeting. @@ -598,7 +652,7 @@ pub async fn stop_recording( speakers: speakers.clone(), duration_secs: (summary.duration_ms / 1000) as i64, recorded: session.retention, - language: None, + language, backend_used: Some(backend_used), model_used: Some(session.model_id.clone()), }, @@ -1338,6 +1392,14 @@ pub async fn list_models() -> WaResult> { Ok(model_catalog::list(&model_id_for(&settings))) } +/// The Settings language dropdown's contents (T8.7, FR-TRX-4) — every +/// whisper.cpp-recognized ISO-639-1 code. "Auto-detect" isn't in this list; +/// the frontend prepends it (maps to omitting `language` at recording start). +#[tauri::command] +pub async fn list_whisper_languages() -> WaResult> { + Ok(crate::transcription::languages::list()) +} + /// The fixed segmentation+embedding pair (T4.7, FR-MODEL-1) — a separate /// command rather than folding into `list_models` because they're a fixed /// installable pair, not an interchangeable-size catalog like whisper's. @@ -1400,6 +1462,10 @@ pub async fn reprocess_transcript( state: State<'_, AppState>, meeting_id: MeetingId, model: String, + // T8.7/FR-TRX-4: `None` reuses whatever language the meeting was + // already recorded/reprocessed with, so re-transcribing with a bigger + // model doesn't silently drop a prior language selection. + language: Option, ) -> WaResult<()> { let model_path = whisper_model_file(&model); if !model_path.exists() { @@ -1416,24 +1482,33 @@ pub async fn reprocess_transcript( )); } + let meeting = state + .store + .get_meeting(&meeting_id) + .await + .map_err(|e| WaError::new("storage", e.to_string()))?; + let settings = load_settings(); let backend = backend_for(&settings); - let segments = tauri::async_runtime::spawn_blocking({ + let requested_language = + normalize_language(language.as_deref().or(meeting.language.as_deref())) + .map(|s| s.to_string()); + let (segments, resolved_language) = tauri::async_runtime::spawn_blocking({ let wav_path = wav_path.clone(); move || { - let (transcriber, _used) = load_transcriber(backend, &model_path)?; - transcriber.transcribe_file(&wav_path) + let (transcriber, _used) = + load_transcriber(backend, &model_path, requested_language.as_deref())?; + let segments = transcriber.transcribe_file(&wav_path)?; + let resolved = transcriber + .detected_language() + .or_else(|| transcriber.effective_language()); + Ok::<_, crate::transcription::TrxError>((segments, resolved)) } }) .await .map_err(|e| WaError::new("transcription", e.to_string()))? .map_err(|e| WaError::new("transcription", e.to_string()))?; - let meeting = state - .store - .get_meeting(&meeting_id) - .await - .map_err(|e| WaError::new("storage", e.to_string()))?; let duration_secs = segments .last() .map(|s| (s.end_ms / 1000) as i64) @@ -1452,7 +1527,7 @@ pub async fn reprocess_transcript( speakers: meeting.speakers.clone(), duration_secs, recorded: meeting.recorded, - language: meeting.language, + language: resolved_language, backend_used: Some(backend.as_str().to_string()), model_used: Some(model), }, @@ -1504,15 +1579,31 @@ pub async fn resume_transcription( )); } + // T8.7/FR-TRX-4: `create_meeting` persisted the originally requested + // language immediately (not just at finalize), so a crash-recovered + // meeting can still honor it here rather than silently reverting to + // auto-detect. + let requested_language = state + .store + .get_meeting(&meeting_id) + .await + .ok() + .and_then(|m| m.language); + // Note: deliberately doesn't emit `recording://state` — that event drives // the main header's single live-session Record/Stop toggle, and reusing // it here would make the header show a phantom "Stop" button for a // recovery run that isn't a `RecordingSession` at all. The "recovering" // badge in the meetings list is this operation's own progress signal. - let segments = tauri::async_runtime::spawn_blocking(move || { - WhisperTranscriber::load(&model_path, BackendId::Cpu) - .and_then(|transcriber| transcriber.transcribe_file(&wav_path)) + let (segments, resolved_language) = tauri::async_runtime::spawn_blocking(move || { + let transcriber = + WhisperTranscriber::load(&model_path, BackendId::Cpu, requested_language.as_deref())?; + let segments = transcriber.transcribe_file(&wav_path)?; + let resolved = transcriber + .detected_language() + .or_else(|| transcriber.effective_language()); + Ok::<_, crate::transcription::TrxError>((segments, resolved)) }) .await .map_err(|e| WaError::new("transcription", e.to_string()))? @@ -1539,7 +1630,7 @@ pub async fn resume_transcription( // The working WAV surviving a crash is the only signal we have // left about intent; keep it rather than silently discard it. recorded: true, - language: None, + language: resolved_language, backend_used: Some(BackendId::Cpu.as_str().to_string()), model_used: Some(model_id), }, @@ -3950,6 +4041,7 @@ mod tests { title: "Test meeting".to_string(), calendar_event_id: None, template_id: None, + language: None, }) .await .unwrap();