iamdoubz/WhispAssist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

feat(transcription): wire language selection through recording/reprocess/recovery (T8.7, M4.2)

Browse Source
This commit is contained in:
iamdoubz
2026-07-07 13:06:00 -05:00
parent 34fff3f7de
commit 7490ff5db4
1 changed files with 116 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/commands.rs
+116 -24
View File
@@ -38,6 +38,12 @@ pub struct StartRecordingArgs {
pub record: bool,
/// Note-template id (Phase 8, T8.1, FR-NOTE-5) — see `notes::built_in_note_templates`.
pub template_id: Option<String>,
/// Per-meeting transcription language override (T8.7, FR-TRX-4): `None`
/// falls back to `Settings.whisper_language`; `None`/`"auto"` (either
/// way) requests auto-detection. Only takes effect with a multilingual
/// model — see `transcription::resolve_language`.
#[serde(default)]
pub language: Option<String>,
}
// ---- File-backed settings (consent/default-retention/storage policy) ----
@@ -54,6 +60,7 @@ fn default_settings() -> Settings {
llm_advanced: serde_json::Value::Null,
preferred_backend: "auto".into(),
whisper_model: crate::paths::DEFAULT_WHISPER_MODEL.to_string(),
whisper_language: None, // auto-detect by default (T8.7, FR-TRX-4)
low_overhead: false,
default_record: false,
consent_acknowledged: false,
@@ -129,6 +136,7 @@ fn backend_for(settings: &Settings) -> BackendId {
fn load_transcriber(
backend: BackendId,
whisper_model: &Path,
language: Option<&str>,
) -> Result<(Box<dyn Transcriber>, BackendId), crate::transcription::TrxError> {
use crate::hardware::{resolve_accel, AccelPath};
@@ -146,7 +154,7 @@ fn load_transcriber(
use crate::transcription::{onnx_models, OnnxTranscriber};
if onnx_models::is_installed(onnx_models::DEFAULT_ONNX_MODEL) {
let dir = onnx_models::model_dir(onnx_models::DEFAULT_ONNX_MODEL);
match OnnxTranscriber::load(&dir, backend) {
match OnnxTranscriber::load(&dir, backend, language) {
Ok(t) => return Ok((Box::new(t), backend)),
Err(e) => tracing::warn!("ONNX engine load failed ({e}); falling back to CPU"),
}
@@ -164,17 +172,29 @@ fn load_transcriber(
AccelPath::WhisperCuda | AccelPath::WhisperVulkan => backend,
_ => BackendId::Cpu,
};
match WhisperTranscriber::load(whisper_model, whisper_backend) {
match WhisperTranscriber::load(whisper_model, whisper_backend, language) {
Ok(t) => Ok((Box::new(t), whisper_backend)),
Err(e) if whisper_backend != BackendId::Cpu => {
tracing::warn!("backend {whisper_backend:?} failed to load ({e}); falling back to CPU");
WhisperTranscriber::load(whisper_model, BackendId::Cpu)
WhisperTranscriber::load(whisper_model, BackendId::Cpu, language)
.map(|t| (Box::new(t) as Box<dyn Transcriber>, BackendId::Cpu))
}
Err(e) => Err(e),
}
}
/// Normalizes a requested language string the same way whisper.cpp itself
/// treats it: an empty string or the literal `"auto"` both mean "no explicit
/// language" — matching `FullParams::set_language`'s own `None`/`Some("auto")`
/// equivalence (T8.7, FR-TRX-4) — so callers can pass either spelling through
/// from settings/args without duplicating this check everywhere.
fn normalize_language(language: Option<&str>) -> Option<&str> {
match language {
Some(l) if !l.is_empty() && !l.eq_ignore_ascii_case("auto") => Some(l),
_ => None,
}
}
fn now_unix() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -276,6 +296,14 @@ pub async fn start_recording(
),
));
}
// T8.7/FR-TRX-4: a per-meeting override wins over the Settings default;
// both spellings of "no explicit language" collapse to `None` here.
let language: Option<String> = normalize_language(
args.language
.as_deref()
.or(settings.whisper_language.as_deref()),
)
.map(|s| s.to_string());
let meeting_id = state
.store
@@ -285,6 +313,7 @@ pub async fn start_recording(
.unwrap_or_else(|| "Untitled meeting".to_string()),
calendar_event_id: args.calendar_event_id,
template_id: args.template_id,
language: language.clone(),
})
.await
.map_err(|e| WaError::new("storage", e.to_string()))?;
@@ -376,6 +405,14 @@ pub async fn start_recording(
let segments_for_worker = segments.clone();
let active_backend: Arc<StdMutex<BackendId>> = Arc::new(StdMutex::new(backend));
let active_backend_for_worker = active_backend.clone();
// T8.7/FR-TRX-4: starts at the requested value, then the worker refines
// it — first to what the loaded engine actually resolved (e.g. forced
// "en" for an English-only model), then to whatever was last
// used/detected once streaming stops. `stop_recording` reads the final
// value after joining this thread.
let language_state: Arc<StdMutex<Option<String>>> = Arc::new(StdMutex::new(language.clone()));
let language_state_for_worker = language_state.clone();
let language_for_worker = language.clone();
let app_for_worker = app.clone();
let meeting_id_for_worker = meeting_id.clone();
let transcription_worker = std::thread::Builder::new()
@@ -385,13 +422,14 @@ pub async fn start_recording(
// fall-through (T3.4/T3.5, FR-HW-4): a GPU/NPU load failure (driver
// issue, OOM, missing model) drops to CPU rather than losing the
// meeting's transcript entirely.
let (transcriber, used) = match load_transcriber(backend, &model_path) {
Ok(pair) => pair,
Err(e) => {
tracing::error!("failed to load any transcriber: {e}");
return;
}
};
let (transcriber, used) =
match load_transcriber(backend, &model_path, language_for_worker.as_deref()) {
Ok(pair) => pair,
Err(e) => {
tracing::error!("failed to load any transcriber: {e}");
return;
}
};
if used != backend {
if let Ok(mut b) = active_backend_for_worker.lock() {
*b = used;
@@ -404,6 +442,9 @@ pub async fn start_recording(
}),
);
}
if let Ok(mut lang) = language_state_for_worker.lock() {
*lang = transcriber.effective_language();
}
run_streaming_worker(transcriber.as_ref(), frame_rx, |segment| {
if let Ok(mut buf) = segments_for_worker.lock() {
buf.push(segment.clone());
@@ -413,6 +454,14 @@ pub async fn start_recording(
serde_json::json!({ "meetingId": meeting_id_for_worker, "segment": segment }),
);
});
// Prefer whatever the engine actually used/detected on its last
// decode over the load-time resolution above — meaningful for
// "auto" mode, where the real answer only exists after decoding.
if let Some(detected) = transcriber.detected_language() {
if let Ok(mut lang) = language_state_for_worker.lock() {
*lang = Some(detected);
}
}
})
.map_err(|e| WaError::new("transcription", e.to_string()))?;
@@ -500,6 +549,7 @@ pub async fn start_recording(
segments,
active_backend,
model_id,
language: language_state,
diarizer,
speaker_names,
});
@@ -586,6 +636,10 @@ pub async fn stop_recording(
.map(|b| b.as_str().to_string())
.unwrap_or_else(|_| BackendId::Cpu.as_str().to_string());
// T8.7/FR-TRX-4: whatever the transcription worker last resolved —
// explicit request, English-only forcing, or auto-detected result.
let language = session.language.lock().ok().and_then(|g| g.clone());
// T2.10: persist transcript.json (via finalize_meeting) and notes.md
// *before* touching the working WAV, so a crash here still leaves a
// recoverable, regenerable meeting.
@@ -598,7 +652,7 @@ pub async fn stop_recording(
speakers: speakers.clone(),
duration_secs: (summary.duration_ms / 1000) as i64,
recorded: session.retention,
language: None,
language,
backend_used: Some(backend_used),
model_used: Some(session.model_id.clone()),
},
@@ -1338,6 +1392,14 @@ pub async fn list_models() -> WaResult<Vec<ModelInfo>> {
Ok(model_catalog::list(&model_id_for(&settings)))
}
/// The Settings language dropdown's contents (T8.7, FR-TRX-4) — every
/// whisper.cpp-recognized ISO-639-1 code. "Auto-detect" isn't in this list;
/// the frontend prepends it (maps to omitting `language` at recording start).
#[tauri::command]
pub async fn list_whisper_languages() -> WaResult<Vec<LanguageOption>> {
Ok(crate::transcription::languages::list())
}
/// The fixed segmentation+embedding pair (T4.7, FR-MODEL-1) — a separate
/// command rather than folding into `list_models` because they're a fixed
/// installable pair, not an interchangeable-size catalog like whisper's.
@@ -1400,6 +1462,10 @@ pub async fn reprocess_transcript(
state: State<'_, AppState>,
meeting_id: MeetingId,
model: String,
// T8.7/FR-TRX-4: `None` reuses whatever language the meeting was
// already recorded/reprocessed with, so re-transcribing with a bigger
// model doesn't silently drop a prior language selection.
language: Option<String>,
) -> WaResult<()> {
let model_path = whisper_model_file(&model);
if !model_path.exists() {
@@ -1416,24 +1482,33 @@ pub async fn reprocess_transcript(
));
}
let meeting = state
.store
.get_meeting(&meeting_id)
.await
.map_err(|e| WaError::new("storage", e.to_string()))?;
let settings = load_settings();
let backend = backend_for(&settings);
let segments = tauri::async_runtime::spawn_blocking({
let requested_language =
normalize_language(language.as_deref().or(meeting.language.as_deref()))
.map(|s| s.to_string());
let (segments, resolved_language) = tauri::async_runtime::spawn_blocking({
let wav_path = wav_path.clone();
move || {
let (transcriber, _used) = load_transcriber(backend, &model_path)?;
transcriber.transcribe_file(&wav_path)
let (transcriber, _used) =
load_transcriber(backend, &model_path, requested_language.as_deref())?;
let segments = transcriber.transcribe_file(&wav_path)?;
let resolved = transcriber
.detected_language()
.or_else(|| transcriber.effective_language());
Ok::<_, crate::transcription::TrxError>((segments, resolved))
}
})
.await
.map_err(|e| WaError::new("transcription", e.to_string()))?
.map_err(|e| WaError::new("transcription", e.to_string()))?;
let meeting = state
.store
.get_meeting(&meeting_id)
.await
.map_err(|e| WaError::new("storage", e.to_string()))?;
let duration_secs = segments
.last()
.map(|s| (s.end_ms / 1000) as i64)
@@ -1452,7 +1527,7 @@ pub async fn reprocess_transcript(
speakers: meeting.speakers.clone(),
duration_secs,
recorded: meeting.recorded,
language: meeting.language,
language: resolved_language,
backend_used: Some(backend.as_str().to_string()),
model_used: Some(model),
},
@@ -1504,15 +1579,31 @@ pub async fn resume_transcription(
));
}
// T8.7/FR-TRX-4: `create_meeting` persisted the originally requested
// language immediately (not just at finalize), so a crash-recovered
// meeting can still honor it here rather than silently reverting to
// auto-detect.
let requested_language = state
.store
.get_meeting(&meeting_id)
.await
.ok()
.and_then(|m| m.language);
// Note: deliberately doesn't emit `recording://state` — that event drives
// the main header's single live-session Record/Stop toggle, and reusing
// it here would make the header show a phantom "Stop" button for a
// recovery run that isn't a `RecordingSession` at all. The "recovering"
// badge in the meetings list is this operation's own progress signal.
let segments = tauri::async_runtime::spawn_blocking(move || {
WhisperTranscriber::load(&model_path, BackendId::Cpu)
.and_then(|transcriber| transcriber.transcribe_file(&wav_path))
let (segments, resolved_language) = tauri::async_runtime::spawn_blocking(move || {
let transcriber =
WhisperTranscriber::load(&model_path, BackendId::Cpu, requested_language.as_deref())?;
let segments = transcriber.transcribe_file(&wav_path)?;
let resolved = transcriber
.detected_language()
.or_else(|| transcriber.effective_language());
Ok::<_, crate::transcription::TrxError>((segments, resolved))
})
.await
.map_err(|e| WaError::new("transcription", e.to_string()))?
@@ -1539,7 +1630,7 @@ pub async fn resume_transcription(
// The working WAV surviving a crash is the only signal we have
// left about intent; keep it rather than silently discard it.
recorded: true,
language: None,
language: resolved_language,
backend_used: Some(BackendId::Cpu.as_str().to_string()),
model_used: Some(model_id),
},
@@ -3950,6 +4041,7 @@ mod tests {
title: "Test meeting".to_string(),
calendar_event_id: None,
template_id: None,
language: None,
})
.await
.unwrap();