iamdoubz/WhispAssist
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

fix(spk): live split attribution + honest model reporting + notes rename rewrite (FR-SPK, T3.4)

Browse Source 
- live diarization tick on split recordings now uses attribute_split (mic
  channel = 'You' from the first pass) instead of whole-mix clustering +
  voiceprint matching, which never reliably surfaced 'You' before stop
- attribute_split assigns via diarization::assign_split (channel-total
  overlap) so the user's speech can't be swallowed by a longer far-side span
- stop_recording overlays live user renames over the automatic name map
- rename_speaker/map_speaker_to_participant rewrite notes.md's baked-in
  **Name:** dialogue tags so the Notes pane follows the Speakers pane
- model_used records the ONNX model when the NPU/DirectML engine is routed
  (was reporting the unused ggml model id) + info log naming the encoder EP
This commit is contained in:
iamdoubz committed 2026-07-15 17:41:28 -05:00
1 parent f24adc3a59
commit 28d9b2988a
1 file changed
+204 -54
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/commands.rs
+204 -54
View File
@@ -120,6 +120,32 @@ fn model_id_for(settings: &Settings) -> String {
}
}
/// The model id to *report* for a session on `backend`: the ONNX engine
/// (NPU/DirectML) ignores the configured ggml model and always runs its own
/// ONNX artifacts, so recording the ggml id would be a lie — e.g. a meeting
/// shown as "medium.en-q5_0 · npu" actually transcribed with ONNX base.en.
/// Mirrors the exact routing condition in `load_transcriber`; if the engine
/// fails to load at runtime the worker falls back to whisper.cpp and the
/// backend is corrected via `hardware://changed`, but this reported model
/// isn't — acceptable for that rare failure path.
fn effective_model_id(backend: BackendId, requested: &str) -> String {
#[cfg(feature = "npu")]
{
use crate::hardware::{resolve_accel, AccelPath};
use crate::transcription::onnx_models;
if matches!(
resolve_accel(backend),
AccelPath::OnnxOpenVino | AccelPath::OnnxDirectML
) && onnx_models::is_installed(onnx_models::DEFAULT_ONNX_MODEL)
{
return format!("{} (onnx)", onnx_models::DEFAULT_ONNX_MODEL);
}
}
#[cfg(not(feature = "npu"))]
let _ = backend;
requested.to_string()
}
/// Picks the backend to transcribe with: "low overhead" always forces CPU;
/// otherwise resolve the user's preferred backend (or auto-detect) against
/// what's actually available (T3.2, T3.5).
@@ -166,7 +192,22 @@ fn load_transcriber(
if onnx_models::is_installed(onnx_models::DEFAULT_ONNX_MODEL) {
let dir = onnx_models::model_dir(onnx_models::DEFAULT_ONNX_MODEL);
match OnnxTranscriber::load(&dir, backend, language) {
Ok(t) => return Ok((Box::new(t), backend)),
Ok(t) => {
// The definitive "is the accelerator actually engaged" line
// (visible with `npm run tauri dev`): the encoder runs on
// the EP below; the autoregressive decoder always runs on
// the CPU EP, which is why Task Manager shows only short
// periodic NPU spikes alongside sustained CPU load.
tracing::info!(
"transcription engine: ONNX {} (encoder EP: {}, decoder: CPU)",
onnx_models::DEFAULT_ONNX_MODEL,
match path {
AccelPath::OnnxOpenVino => "OpenVINO/NPU",
_ => "DirectML/GPU",
}
);
return Ok((Box::new(t), backend));
}
Err(e) => tracing::warn!("ONNX engine load failed ({e}); falling back to CPU"),
}
} else {
@@ -257,7 +298,8 @@ fn diarizer_from_installed_models() -> Option<SherpaDiarizer> {
/// Split attribution (FR-SPK, supersedes `phase3_attribute`): diarize the far
/// side (right/loopback channel) into `Speaker N`, take "You" straight from
/// left-channel (mic) voice activity, merge + assign + name. Shared by
/// left-channel (mic) voice activity, attribute per channel
/// (`diarization::assign_split`) + name. Shared by
/// `stop_recording` and `reprocess_transcript`; both read it back from the file
/// so they agree. `None` if the far-side pass fails (caller falls back).
async fn attribute_split(
@@ -268,16 +310,19 @@ async fn attribute_split(
let diarizer_for_task = diarizer.clone();
let result = tauri::async_runtime::spawn_blocking(move || {
// Right channel = loopback/far side; diarize it alone (mic never in it).
// Its VAD is the "was the far side talking at all" evidence assign_split
// weighs against the mic channel.
let far = crate::audio::read_wav_channel_16k(&wav_path, 1).map_err(|e| e.to_string())?;
let far_vad = crate::audio::vad_spans(&far);
let far_spans = diarizer_for_task
.diarize_samples(far)
.map_err(|e| e.to_string())?;
// Left channel = mic; its voice activity is "You".
let mic = crate::audio::read_wav_channel_16k(&wav_path, 0).map_err(|e| e.to_string())?;
Ok::<_, String>((far_spans, crate::audio::vad_spans(&mic)))
Ok::<_, String>((far_spans, far_vad, crate::audio::vad_spans(&mic)))
})
.await;
let (far_spans, you_spans) = match result {
let (far_spans, far_vad, you_spans) = match result {
Ok(Ok(v)) => v,
Ok(Err(e)) => {
tracing::warn!("split attribution failed: {e}");
@@ -288,6 +333,8 @@ async fn attribute_split(
return None;
}
};
crate::diarization::assign_split(segments, &you_spans, &far_vad, &far_spans);
// Merged span timeline, only for first-appearance naming order below.
let mut spans: Vec<SpeakerSpan> = you_spans
.iter()
.map(|&(start_ms, end_ms)| SpeakerSpan {
@@ -298,7 +345,6 @@ async fn attribute_split(
.collect();
spans.extend(far_spans);
spans.sort_by_key(|s| s.start_ms);
diarizer.assign(segments, &spans);
// Uniform naming: "You" -> "You", far speakers -> "Speaker 2", "Speaker 3"….
let labels = crate::diarization::voiceprint::first_appearance_order(&spans);
Some(crate::diarization::voiceprint::build_name_map(&labels, "You"))
@@ -574,7 +620,8 @@ pub async fn start_recording(
let wav_path_for_diar = wav_path.clone();
let segments_for_diar = segments.clone();
let names_for_diar = speaker_names.clone();
let voice_sample_for_diar = mic_voice_sample.clone();
// Same condition as `audio_layout` below: mic on → split stereo file.
let split_layout = settings.microphone_enabled;
tauri::async_runtime::spawn(async move {
// ponytail: reprocesses the whole recording-so-far each tick
// rather than incremental/windowed segmentation — sherpa-onnx's
@@ -597,56 +644,83 @@ pub async fn start_recording(
break; // recording stopped (or a new one started) — nothing left to do
}
let diarizer_for_pass = diarizer.clone();
let wav_path = wav_path_for_diar.clone();
let spans = tauri::async_runtime::spawn_blocking(move || {
diarizer_for_pass.diarize(&wav_path)
})
.await;
let spans = match spans {
Ok(Ok(spans)) => spans,
Ok(Err(e)) => {
tracing::warn!("live diarization pass failed: {e}");
continue;
// Split recording: the exact same channel-based attribution as
// the post-stop pass — mic (left) is always "You", the far side
// (right) is diarized alone. The old whole-mix pass + voiceprint
// cosine match never reliably showed "You" live (clusters over
// the summed mix reshuffle every tick and the match often missed
// its threshold), so "You" only appeared after stop.
let (speakers, changed) = if split_layout {
let mut snapshot = segments_for_diar
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone();
if attribute_split(
diarizer.clone(),
wav_path_for_diar.clone(),
&mut snapshot,
)
.await
.map(|auto_names| {
let mut names = names_for_diar.lock().unwrap_or_else(|e| e.into_inner());
// Never overwrite a name already set (user rename or a
// prior pass) — same guard as the post-stop pass.
for (label, name) in auto_names {
names.entry(label).or_insert(name);
}
})
.is_none()
{
continue; // pass failed (already logged); retry next tick
}
Err(e) => {
tracing::warn!("live diarization task failed: {e}");
continue;
}
};
let (speakers, changed) = {
let relabeled: HashMap<u64, String> = snapshot
.into_iter()
.map(|s| (s.id, s.speaker))
.collect();
let mut segs = segments_for_diar.lock().unwrap_or_else(|e| e.into_inner());
// Snapshot prior labels so only segments whose speaker
// actually changed this pass get re-emitted (ids are stable;
// the frontend replaces by id).
// the frontend replaces by id). Segments committed while the
// pass ran keep their placeholder until the next tick.
let before: HashMap<u64, String> =
segs.iter().map(|s| (s.id, s.speaker.clone())).collect();
for seg in segs.iter_mut() {
if let Some(speaker) = relabeled.get(&seg.id) {
seg.speaker = speaker.clone();
}
}
let names = names_for_diar.lock().unwrap_or_else(|e| e.into_inner());
let changed: Vec<TranscriptSegment> = segs
.iter()
.filter(|s| before.get(&s.id) != Some(&s.speaker))
.cloned()
.collect();
(speaker_infos_from_segments(&segs, &names), changed)
} else {
// Summed recording (mic off): whole-signal pass. No mic
// channel → no live "You" (the mic voice sample doesn't
// exist either), matching the post-stop behavior.
let diarizer_for_pass = diarizer.clone();
let wav_path = wav_path_for_diar.clone();
let spans = tauri::async_runtime::spawn_blocking(move || {
diarizer_for_pass.diarize(&wav_path)
})
.await;
let spans = match spans {
Ok(Ok(spans)) => spans,
Ok(Err(e)) => {
tracing::warn!("live diarization pass failed: {e}");
continue;
}
Err(e) => {
tracing::warn!("live diarization task failed: {e}");
continue;
}
};
let mut segs = segments_for_diar.lock().unwrap_or_else(|e| e.into_inner());
let before: HashMap<u64, String> =
segs.iter().map(|s| (s.id, s.speaker.clone())).collect();
diarizer.assign(&mut segs, &spans);
// Live "You": match the mic voiceprint against this pass's
// clusters. Clusters re-shuffle every tick so match every
// tick; never overwrite a name already set (user rename or a
// prior pass) — same guard as the post-stop pass.
if let Some(voice_sample) = &voice_sample_for_diar {
let mic_samples = voice_sample.samples();
match crate::diarization::voiceprint::match_mic_speaker(
&diarization_embedding_model_file(),
&mic_samples,
&wav_path_for_diar,
&spans,
) {
Ok(auto_names) => {
let mut names =
names_for_diar.lock().unwrap_or_else(|e| e.into_inner());
for (label, name) in auto_names {
names.entry(label).or_insert(name);
}
}
Err(e) => tracing::warn!("live voiceprint match failed: {e}"),
}
}
let names = names_for_diar.lock().unwrap_or_else(|e| e.into_inner());
let changed: Vec<TranscriptSegment> = segs
.iter()
@@ -682,7 +756,10 @@ pub async fn start_recording(
transcription_worker,
segments,
active_backend,
model_id,
// Report the model the routed engine will actually run (the ONNX
// engine ignores the configured ggml model), so the meeting's
// "transcribed with … · npu" line is truthful (T3.4/T3.5).
model_id: effective_model_id(backend, &model_id),
language: language_state,
diarizer,
speaker_names,
@@ -761,9 +838,17 @@ pub async fn stop_recording(
let mut attributed = false;
if session.audio_layout == "split" {
if let Some(diarizer) = session.diarizer.clone() {
if let Some(names) =
if let Some(mut names) =
attribute_split(diarizer, session.wav_path.clone(), &mut segments).await
{
// Renames made *during* the recording are user-authored — they
// win over the automatic "You"/"Speaker N" defaults instead of
// being silently dropped at finalize.
if let Ok(user_names) = session.speaker_names.lock() {
for (label, name) in user_names.iter() {
names.insert(label.clone(), name.clone());
}
}
speaker_names = names;
attributed = true;
}
@@ -1260,6 +1345,55 @@ async fn refresh_notes_and_notify(
Ok(meeting.speakers)
}
/// `notes.md` bakes display names into its `**Name:**` dialogue tags at
/// finalize, so a post-meeting rename must rewrite them or the Notes pane
/// (and every export) keeps the old name forever. Targeted tag replace, not
/// a regenerate, so the user's own edits to notes.md are preserved.
async fn rename_speaker_in_notes(
state: &State<'_, AppState>,
meeting_id: &MeetingId,
old_name: &str,
new_name: &str,
) -> WaResult<()> {
if old_name == new_name || new_name.is_empty() {
return Ok(());
}
let meeting = state
.store
.get_meeting(meeting_id)
.await
.map_err(|e| WaError::new("storage", e.to_string()))?;
let old_tag = format!("**{old_name}:**");
if meeting.notes_markdown.contains(&old_tag) {
let updated = meeting
.notes_markdown
.replace(&old_tag, &format!("**{new_name}:**"));
state
.store
.update_notes(meeting_id, &updated)
.await
.map_err(|e| WaError::new("storage", e.to_string()))?;
}
Ok(())
}
/// A speaker's current name as notes.md renders it: display name, else the
/// raw label. `None` if the meeting/speaker can't be read (nothing to rewrite).
async fn current_speaker_name(
state: &State<'_, AppState>,
meeting_id: &MeetingId,
label: &str,
) -> Option<String> {
let meeting = state.store.get_meeting(meeting_id).await.ok()?;
let speaker = meeting.speakers.iter().find(|s| s.label == label)?;
Some(
speaker
.display_name
.clone()
.unwrap_or_else(|| label.to_string()),
)
}
/// Name a speaker; applies to that speaker's past & future segments (T4.4,
/// FR-SPK-2). Segments only ever carry the internal label ("S1"…) — never
/// rewritten — so persisting the label→name mapping here is enough to cover
@@ -1274,6 +1408,9 @@ pub async fn rename_speaker(
label: String,
name: String,
) -> WaResult<()> {
// Resolve the name notes.md currently shows *before* the rename lands, so
// the finalized-meeting branch below can rewrite its dialogue tags.
let old_name = current_speaker_name(&state, &meeting_id, &label).await;
state
.store
.rename_speaker(&meeting_id, &label, &name)
@@ -1309,6 +1446,9 @@ pub async fn rename_speaker(
}
None => {
drop(guard);
if let Some(old_name) = old_name {
rename_speaker_in_notes(&state, &meeting_id, &old_name, &name).await?;
}
refresh_notes_and_notify(&app, &state, &meeting_id).await?;
}
}
@@ -1371,12 +1511,22 @@ pub async fn map_speaker_to_participant(
}
drop(guard);
let old_name = current_speaker_name(&state, &meeting_id, &label).await;
state
.store
.map_speaker_to_participant(&meeting_id, &label, &participant_id)
.await
.map_err(|e| WaError::new("storage", e.to_string()))?;
refresh_notes_and_notify(&app, &state, &meeting_id).await?;
let speakers = refresh_notes_and_notify(&app, &state, &meeting_id).await?;
// Rewrite notes.md's baked-in dialogue tags to the participant's name,
// same as a free-text rename (the Notes pane must follow the Speakers pane).
let new_name = speakers
.iter()
.find(|s| s.label == label)
.and_then(|s| s.display_name.clone());
if let (Some(old_name), Some(new_name)) = (old_name, new_name) {
rename_speaker_in_notes(&state, &meeting_id, &old_name, &new_name).await?;
}
Ok(())
}
@@ -2209,7 +2359,7 @@ pub async fn reprocess_transcript(
recorded: meeting.recorded,
language: resolved_language,
backend_used: Some(backend.as_str().to_string()),
model_used: Some(model),
model_used: Some(effective_model_id(backend, &model)),
audio_layout: None, // reprocess preserves the recorded layout
},
)
@@ -2449,7 +2599,7 @@ async fn run_import_pipeline(
recorded: true, // the imported WAV is the recording — keep it
language: resolved_language,
backend_used: Some(backend.as_str().to_string()),
model_used: Some(model_id.clone()),
model_used: Some(effective_model_id(backend, &model_id)),
audio_layout: Some("summed".to_string()), // single-source import
},
)