Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2c8958da01 | ||
|
|
f2c945838c | ||
|
|
0e0f3a09f1 | ||
|
|
3d183fdd7c | ||
|
|
32bb4057ec | ||
|
|
90a9cbd476 | ||
|
|
5bd76b80e8 | ||
|
|
a1d71d1c73 | ||
|
|
e287fa5ebe | ||
|
|
534f5ca350 | ||
|
|
1e99c4c747 | ||
|
|
5e5f61f3b7 | ||
|
|
dec9251e7b | ||
|
|
d4e2ab494c | ||
|
|
845903a772 | ||
|
|
998de62df2 | ||
|
|
e449c194ef | ||
|
|
6cea767dec | ||
|
|
60ae97c771 | ||
|
|
d8587b8e93 | ||
|
|
695a12dc2d | ||
|
|
6385e38b33 | ||
|
|
0b5a85f461 | ||
|
|
fd9311c482 | ||
|
|
39294adcbc | ||
|
|
fee18ebc59 | ||
|
|
51f5ff085c | ||
|
|
6d4204fb22 | ||
|
|
185489fa18 | ||
|
|
41c2c79797 | ||
|
|
6dd6c3c69f | ||
|
|
9468781218 | ||
|
|
b39352dbff | ||
|
|
0d263637df | ||
|
|
bb6e222101 | ||
|
|
85ed3b5587 | ||
|
|
68d692fbe8 | ||
|
|
fa664e5147 | ||
|
|
b5c99621b5 | ||
|
|
db52b9e812 | ||
|
|
319e7a31cc | ||
|
|
25011d0f27 | ||
|
|
7733e86cfd | ||
|
|
a481f5dcbf | ||
|
|
bd6519b65e |
@@ -84,10 +84,14 @@ issues from feature briefs.
|
||||
4. Provisional speaker turns (from cheap segmentation) tag segments live; the user may name a
|
||||
speaker, applied immediately to past/future segments (FR-SPK-2).
|
||||
5. The notes view renders Markdown that updates as segments arrive (FR-NOTE-1, FR-NOTE-6).
|
||||
6. On `stop_recording`: finalize the WAV; run full `diarization`, align speaker IDs to segments,
|
||||
apply name mappings; persist transcript JSON + metadata via `storage` (FR-STORE-1). If "Record
|
||||
this meeting" is **off**, delete the working WAV **after** the transcript is finalized; if **on**,
|
||||
keep it as `audio.wav` (ADR-0009, FR-REC-1/4).
|
||||
6. On `stop_recording`: finalize the WAV; run `diarization`, align speaker IDs to segments,
|
||||
apply name mappings; persist transcript JSON + metadata via `storage` (FR-STORE-1). When the mic
|
||||
is on, `audio.wav` is a **dual-channel split** (left = mic, right = loopback; ADR-0005 Phase 3.5,
|
||||
FR-SPK), so attribution is **per-stream**: sherpa clusters the **right channel only** →
|
||||
`Speaker N`, and "You" comes from **left-channel** voice activity. Blind whole-signal clustering
|
||||
+ voiceprint is the mic-off (`summed`) fallback. If "Record this meeting" is **off**, delete the
|
||||
working WAV **after** the transcript is finalized; if **on**, keep it as `audio.wav` (ADR-0009,
|
||||
FR-REC-1/4).
|
||||
7. If an LLM provider is configured: `llm` builds a prompt (transcript + metadata + template),
|
||||
streams a summary/decisions/action-items into the summary panel (FR-LLM-2/4).
|
||||
8. Action items are parsed and presented for confirmation; confirmed ones persist and may raise
|
||||
|
||||
@@ -14,7 +14,8 @@ Default root: `%LOCALAPPDATA%\WhispAssist\` (user-configurable, FR-STORE-2).
|
||||
│ └── spk-eres2net.onnx
|
||||
└── meetings\
|
||||
└── <meeting_id>\ # one folder per meeting (uuid)
|
||||
├── audio.wav # canonical recording — present ONLY if "Record" was on (ADR-0009)
|
||||
├── audio.wav # canonical recording — present ONLY if "Record" was on (ADR-0009). Mic on → stereo split (L=mic/"You", R=loopback/"Speaker", `audio_layout=split`); mic off → summed. Playback folds to mono; export folds to dual-mono (ADR-0005 Phase 3.5)
|
||||
├── voiceprint.wav # mic-only 16kHz sample for "You" re-matching on reprocess of a SUMMED recording (FR-SPK); lives/dies with audio.wav
|
||||
├── transcript.json # canonical transcript (segments+speakers+timings)
|
||||
├── manual_notes.json # raw user-authored notes captured live during recording
|
||||
├── notes.md # the final notes document: manual notes + transcript, merged at finalize
|
||||
@@ -86,6 +87,7 @@ CREATE TABLE meetings (
|
||||
language TEXT, -- detected/selected language code
|
||||
backend_used TEXT, -- npu|nvidia|amd|intel|cpu
|
||||
model_used TEXT, -- e.g. whisper-base
|
||||
audio_layout TEXT, -- split|summed|NULL; how audio.wav's channels are laid out (0009, ADR-0005 Phase 3.5)
|
||||
calendar_event_id TEXT, -- FK -> calendar_events.id (nullable)
|
||||
template_id TEXT, -- note-template id (T8.1, FR-NOTE-5); catalog is
|
||||
-- a built-in Rust list (notes::templates), not a table
|
||||
|
||||
@@ -183,9 +183,9 @@ privacy_self_check(): {
|
||||
"recording://state" { meetingId, state: "recording"|"paused"|"stopped"|"cancelled", elapsedMs }
|
||||
"recording://level" { meetingId, rms: number, peak: number } // waveform (FR-CAP-5)
|
||||
"recording://device" { meetingId, recovered: boolean, message: string } // capture device change (FR-CAP-6)
|
||||
"transcript://segment" { meetingId, segment: TranscriptSegment } // live segments (FR-TRX-2)
|
||||
"transcript://segment" { meetingId, segment: TranscriptSegment } // live segments (FR-TRX-2); may re-emit a committed segment with a refined `speaker` — replace by `segment.id`
|
||||
"transcript://finalized" { meetingId, segmentCount }
|
||||
"diarization://updated" { meetingId, speakers: SpeakerInfo[] } // after post-pass (FR-SPK)
|
||||
"diarization://updated" { meetingId, speakers: SpeakerInfo[] } // post-pass AND live 15s provisional passes (FR-SPK); carries "You" once the mic voiceprint matches
|
||||
"llm://token" { meetingId, text } // streamed summary (FR-LLM-4)
|
||||
"llm://done" { meetingId, summary: SummaryFile } // full summary.json contents, not just a pointer
|
||||
"model://progress" { id, receivedBytes, totalBytes }
|
||||
|
||||
@@ -32,6 +32,54 @@ labels in the post-meeting pass; this keeps latency low while improving final ac
|
||||
- Speaker IDs (`S1`, `S2`, …) are internal and stable per meeting; name mappings live in the DB
|
||||
and are applied at render/export time, never destructively rewritten onto segments.
|
||||
|
||||
### Phase 3 refinement — per-stream "You" attribution (FR-SPK, 2026-07-14)
|
||||
|
||||
The single-pass-over-the-whole-recording model above blind-clusters a **summed mono** signal
|
||||
(mic + loopback), then guesses which cluster is the user via a voiceprint match. In practice that
|
||||
clustering is unreliable in both directions on the summed signal — it over-split a 2-speaker call
|
||||
into 83 clusters at one threshold and merged two clearly distinct voices into one at another — and
|
||||
the voiceprint can only *label* a cluster, never *create* the separation.
|
||||
|
||||
So when the **microphone is enabled**, WA no longer relies on clustering to find the user:
|
||||
|
||||
- A live **mic-activity timeline** (`audio::MicActivity`) records, per 100 ms of `audio.wav` frame
|
||||
time, when the mic was speech-level — captured in the loopback writer, the one place the mic and
|
||||
loopback exist separately in the recording's own timebase. Those ranges become **"You"** spans
|
||||
directly (no clustering, no embedding).
|
||||
- sherpa clustering is then run over the recording with the "You" ranges **masked out** (zeroed),
|
||||
so it only ever splits the **far side** into `Speaker N`. The mic can never pollute or merge into
|
||||
a far-side cluster.
|
||||
- The timeline is persisted as `mic_activity.json` (retained meetings) so `reprocess_transcript`
|
||||
re-attributes identically without the live capture.
|
||||
|
||||
The original blind-clustering + voiceprint pass **remains the fallback** for mic-off recordings and
|
||||
imports (no timeline). Naming stays uniform (`build_name_map`: "You", then `Speaker 2…`). Segment
|
||||
IDs and the names-in-DB rule are unchanged; only the *source* of the spans changes.
|
||||
|
||||
### Phase 3.5 refinement — dual-channel capture (supersedes Phase 3's timeline, 2026-07-14)
|
||||
|
||||
The Phase 3 timeline above reconstructed the mic/far-side split *after the fact* from a summed-mono
|
||||
`audio.wav` plus a `mic_activity.json` sidecar, masking the mic ranges before clustering. That was
|
||||
reliable at stop but **fragile on reprocess** (re-aligning a sidecar against a mono mix — it
|
||||
collapsed to a single speaker) and carried a parallel persistence path.
|
||||
|
||||
**Decision:** when the mic is enabled, record `audio.wav` as **stereo with the streams separated —
|
||||
left = microphone ("You"), right = system/loopback ("Speaker")** instead of summing them. The
|
||||
separation is then intrinsic to the recording:
|
||||
|
||||
- Diarization runs on the **right channel only** → `Speaker N`; "You" comes from **left-channel**
|
||||
voice activity (`vad_spans`). The mic is never clustered, by construction.
|
||||
- Reprocess recomputes both from the file — no sidecar, no masking; stop and reprocess agree.
|
||||
- Transcription still downmixes (L+R) to the same summed mono; playback folds to mono and bundle
|
||||
export folds to dual-mono so shared/played audio is normal.
|
||||
- A `meetings.audio_layout` flag (`summed` | `split`, carried in the bundle manifest) distinguishes
|
||||
recordings; existing `summed` recordings keep the blind-clustering + voiceprint path. On a stereo
|
||||
render endpoint this is **size-neutral** (the file was already 2ch with the mic summed into both).
|
||||
|
||||
This **retires** `MicActivity` / `mic_activity.json` / masked diarization from Phase 3 (see the
|
||||
retirement note at `attribute_split` in `commands.rs`); the capture-silence handling,
|
||||
`diarize_samples`, and `build_name_map` naming are retained. Segment IDs and names-in-DB unchanged.
|
||||
|
||||
## Revisit if
|
||||
A single model gives joint ASR + diarization with better accuracy, or whisper.cpp gains
|
||||
production diarization.
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
# Phase 3 design sketch: per-stream speaker attribution (FR-SPK)
|
||||
|
||||
Status: **signed off & in progress**, 2026-07-13. Deepens Phase 3 of
|
||||
[`2026-07-13-diarization-speaker-accuracy.md`](./2026-07-13-diarization-speaker-accuracy.md).
|
||||
The §§1–8 below are the original sketch (ADR-0005 review + alternatives); the **finalized
|
||||
design** the user signed off on is at the top and supersedes the RMS-dominance heuristic.
|
||||
|
||||
---
|
||||
|
||||
## Finalized design (signed off 2026-07-13)
|
||||
|
||||
Simpler than the sketch's dominance heuristic, because the user records on **headphones** so the
|
||||
mic never hears the far side (no bleed):
|
||||
|
||||
> **The microphone stream is "You" by definition. Diarization runs on the loopback (speaker)
|
||||
> audio only, never the mic. Any loopback speaker is "Speaker N".**
|
||||
|
||||
Mechanism, all in **`audio.wav` frame time** (so You-spans and sherpa-spans share one clock):
|
||||
|
||||
1. **Mic-activity timeline** — captured in the loopback writer, the one place mic and loopback
|
||||
exist separately in `audio.wav`'s timebase (`capture_loop`, where `bridge.pull()` is folded
|
||||
in). Per 100 ms, flag whether the pulled mic chunk was speech-level (reuse `VOICE_ENERGY_FLOOR`).
|
||||
Held in a shared `MicActivity` (sibling to `VoiceSample`); persisted for reprocess.
|
||||
2. **At stop / reprocess:**
|
||||
- Collapse active windows → **"You" spans** (dropping sub-`MIN_SPAN_MS` blips).
|
||||
- **Mask** the You ranges out of `audio.wav` (zero them) and run sherpa on the remainder → far
|
||||
side splits into **"Speaker N"** spans. Clustering never sees the mic, so the mic can't
|
||||
pollute or merge with a far-side cluster.
|
||||
- Merge span lists; `assign_by_overlap` unchanged.
|
||||
3. **Voiceprint (Phase 1/2) becomes the fallback** for meetings with no timeline (imports, older
|
||||
recordings). Live-tick voiceprint "You" is retired once the timeline drives labels.
|
||||
|
||||
Timebase note: with the capture-silence fix (FR-CAP-7) `audio.wav` is now continuous, so the
|
||||
frame-indexed timeline aligns cleanly with what sherpa reads back via `read_wav_mono_16k`.
|
||||
|
||||
Implementation status (all landed 2026-07-13/14): **(a)** `MicActivity` struct + capture wiring —
|
||||
done; **(b)** stop-path attribution + masked diarize (`phase3_attribute`) — done; **(c)** persist
|
||||
`mic_activity.json` + reprocess uses it (imports fall back, no timeline) — done; **(d)** ADR-0005 +
|
||||
`docs/02`/`03` amended — done. **Not yet done:** empirical user test; and the *live* 15s tick still
|
||||
uses the Phase 2 voiceprint, not the timeline (a later polish so live labels match the stop result).
|
||||
|
||||
---
|
||||
|
||||
_Original sketch (ADR-0005 review, risks, alternatives) follows._
|
||||
|
||||
Phases 0–2 shipped: cluster explosion tamed (0.7 + 700 ms floor), reprocess re-diarizes with a
|
||||
persistent `voiceprint.wav`, and live "You" reaches the UI. Those all still treat diarization as
|
||||
**blind clustering of a summed mono signal**, then guess which cluster is the mic. Phase 3
|
||||
removes the guess for the mic side by using information WA throws away today.
|
||||
|
||||
---
|
||||
|
||||
## 1. ADR-0005 review — what it commits us to
|
||||
|
||||
ADR-0005 (Accepted, 2026-06-30) decides: sherpa-onnx offline diarization (pyannote segmentation
|
||||
+ ERes2Net embedding + clustering), behind the `diarization::Diarizer` trait, run as a
|
||||
**post-processing pass over recorded audio**, aligned to whisper segments by **timestamp
|
||||
overlap**. Speaker IDs (`S1`…) are internal/stable; names map in the DB, applied at render time,
|
||||
**never rewritten onto segments**.
|
||||
|
||||
What Phase 3 must respect vs. what it may change:
|
||||
|
||||
- **Keeps (contract-level):** the `Diarizer` trait, post-pass-over-audio model, `SpeakerSpan`
|
||||
→ segment alignment by overlap (`assign_by_overlap`), stable internal labels, names-in-DB.
|
||||
Phase 3 produces *more* `SpeakerSpan`s from a better source; it does not rewrite segments or
|
||||
move naming.
|
||||
- **Bends (needs the ADR noted/updated):** ADR-0005 assumes one clustering pass over "the
|
||||
recorded audio." Phase 3 introduces a **second, non-clustered source of spans** (the mic
|
||||
activity timeline) and restricts clustering to the **far-side** audio. That is new enough to
|
||||
warrant an ADR amendment or a short ADR-0005a, because a future reader will otherwise expect
|
||||
all spans to come from sherpa.
|
||||
- **ADR's "Revisit if"** (joint ASR+diarization model, or whisper.cpp diarization) is unrelated
|
||||
— Phase 3 is orthogonal and does not trigger it.
|
||||
|
||||
Conclusion: Phase 3 fits inside the trait and the overlap-alignment contract. The only doc debt
|
||||
is recording that "mic-dominant spans bypass clustering," which is a genuine deviation from
|
||||
ADR-0005's single-source assumption. **Recommend: amend ADR-0005** (Consequences section) rather
|
||||
than a new ADR — same decision, refined.
|
||||
|
||||
---
|
||||
|
||||
## 2. The asset we currently discard
|
||||
|
||||
When the mic is enabled (FR-CAP-7), `start_recording` (`commands.rs:342`) wires two independent
|
||||
16 kHz-mono streams into `spawn_mixer` (`audio/mod.rs:1020`). The `Mixer` (`audio/mod.rs:979`)
|
||||
holds `loopback` and `mic` **time-aligned in the same buffer** and sums them sample-for-sample
|
||||
into the transcript stream — then the per-stream identity is **gone**. Diarization later reads
|
||||
`audio.wav` (the summed signal) and has to reverse-engineer which cluster was the mic. That
|
||||
reversal is the entire reason `voiceprint.rs` exists.
|
||||
|
||||
Two facts make this cheap to exploit:
|
||||
|
||||
1. **Per-stream RMS already exists.** `audio_level(mono, mic)` (`audio/mod.rs:522`) runs on every
|
||||
WASAPI chunk for *both* directions (`is_loopback` at :564, emitted at :678). We already know,
|
||||
per ~10 ms chunk, how loud each side is. We just don't persist it against time.
|
||||
2. **The two streams are already aligned at the mixer**, in the **same timebase the transcript
|
||||
segments derive from** (the streaming worker consumes the mixer's summed output; segment
|
||||
timestamps are cumulative-samples-fed / 16). So a timeline indexed by cumulative mixer-output
|
||||
samples shares the segments' clock — tighter than sherpa spans, which come from `audio.wav`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Core idea
|
||||
|
||||
> The mic stream is a **known** speaker. Don't cluster to find it — record when it's dominant,
|
||||
> call those spans "You", and run sherpa only on the **far-side** audio to split the *other*
|
||||
> participants.
|
||||
|
||||
Pipeline at stop (mic enabled + retained):
|
||||
|
||||
```
|
||||
mixer ─┬─► loopback samples ──► [far-side WAV] ──► sherpa cluster ──► Speaker 2..N spans
|
||||
│ │
|
||||
└─► mic vs loopback RMS per 100 ms ──► mic-dominant ranges ──► "You" spans
|
||||
│
|
||||
merge span lists ──► assign_by_overlap ──► segments
|
||||
```
|
||||
|
||||
The mic-dominant spans need no embedding, no clustering, no voiceprint — they are attribution by
|
||||
construction. Voiceprint (Phases 1–2) stays as the **fallback** for meetings without a timeline
|
||||
(imports, pre-Phase-3 recordings, mic-disabled meetings).
|
||||
|
||||
---
|
||||
|
||||
## 4. The two real risks (why this needs sign-off, not just a ticket)
|
||||
|
||||
### 4a. Acoustic bleed — the mic is not purely "You"
|
||||
|
||||
Without echo cancellation (WA has none — fully local, minimal), a user on **speakers** (not
|
||||
headphones) has the far side playing into the room and back into the mic. Naive "mic has signal →
|
||||
You" would attribute the far side to the user whenever the far side is loud.
|
||||
|
||||
Mitigation (this is why the parent plan says *mic-**dominant**,* not *mic-active*): a range is
|
||||
"You" only when **mic RMS meaningfully exceeds loopback RMS** in that window (e.g.
|
||||
`mic_rms > k · loopback_rms` with `k ≈ 2`, plus an absolute mic-VAD floor so silence isn't
|
||||
"You"). When both are comparably loud → treat as far-side/ambiguous, let sherpa/overlap decide.
|
||||
This degrades gracefully: headphone users get near-perfect mic isolation; speaker users get
|
||||
"You" only on clear self-speech and fall back to the old behavior during overlap. **`k` is a
|
||||
calibration knob, not a constant to bury** — real rooms differ.
|
||||
|
||||
### 4b. Timebase — three clocks, currently reconciled by tolerance
|
||||
|
||||
- **Segments** are in *mixer-output* time (streaming worker over the summed stream).
|
||||
- **sherpa spans** are in *`audio.wav`* time (loopback thread's byte-accurate WAV, mic bridged in
|
||||
at native rate then resampled to 16 kHz on read).
|
||||
- The **new mic-timeline** would be in *mixer-output* time.
|
||||
|
||||
Today `assign_by_overlap` already bridges segments↔sherpa-spans across the first two clocks and
|
||||
tolerates the small skew. Phase 3 merges a *third* source. The mic-timeline is actually the
|
||||
**closest** to segment time (shared origin), so aligning You-spans to segments is *more* reliable
|
||||
than sherpa's. The residual risk is only that You-spans (mixer time) and Speaker-2..N spans
|
||||
(`audio.wav` time) coexist in one merged list with a small relative skew at the boundaries. In
|
||||
practice that costs at most a fraction of a `MIN_SPAN_MS` at each turn boundary — below segment
|
||||
granularity. **Verification item, not a blocker.** If it proves visible, the clean fix is to run
|
||||
the far-side sherpa pass over the mixer's *loopback lane* buffered to a WAV rather than over
|
||||
`audio.wav`, putting everything on one clock; heavier, so deferred until measured.
|
||||
|
||||
---
|
||||
|
||||
## 5. Concrete design
|
||||
|
||||
**Capture (live, near-zero cost).** Add a mic-activity accumulator the mixer writes to. In
|
||||
`Mixer::drain_ready` (or the `mixer_loop`), for each emitted window compute `mic_rms` and
|
||||
`loopback_rms` over the aligned overlap (samples are already in hand there) and append a coarse
|
||||
flag per 100 ms:
|
||||
|
||||
```rust
|
||||
// audio/mod.rs — new, alongside Mixer
|
||||
struct MicActivity { flags: Vec<bool>, /* one per 100 ms of mixer output */ }
|
||||
// ponytail: Vec<bool> at 10 Hz = 10 bytes/s ≈ 36 KB/hour. No ring buffer, no config.
|
||||
```
|
||||
|
||||
Expose it via the same `Arc` sharing pattern `VoiceSample` uses so `stop_recording` can read it.
|
||||
Reuse `audio_level`'s RMS math — do **not** add a second RMS implementation.
|
||||
|
||||
**Persist (retained meetings).** At stop, alongside `voiceprint.wav`, write the timeline as
|
||||
`mic_activity.json` (or pack into an existing artifact) so **reprocess** can reuse it — same
|
||||
ADR-0009 retention/consent gate as `voiceprint.wav`, same lives-and-dies-with-`audio.wav` rule
|
||||
(delete it everywhere `audio.wav` is deleted). Tiny, plaintext-or-sealed to match.
|
||||
|
||||
**Attribute (stop + reprocess).**
|
||||
1. Collapse the flag timeline into `You` `SpeakerSpan`s (merge adjacent flags, drop runs
|
||||
`< MIN_SPAN_MS`, reuse Phase 0's floor).
|
||||
2. Build a **far-side-only** 16 kHz signal: zero out (or excise) mic-dominant ranges from the
|
||||
loopback lane, run the existing `SherpaDiarizer::diarize` over it → `Speaker 2..N` spans
|
||||
(label numbering starts at 2, mirroring `voiceprint::build_name_map`).
|
||||
3. Merge the two span lists; `assign_by_overlap` unchanged.
|
||||
4. Fallback chain: no timeline → today's Phase-1/2 voiceprint path; no models → raw `S1`.
|
||||
|
||||
**Contract/doc changes.**
|
||||
- Amend **ADR-0005** Consequences: "mic-dominant spans are attributed directly from a per-stream
|
||||
activity timeline and bypass clustering; sherpa runs on the far-side residual only."
|
||||
- `docs/02-architecture.md` data-flow: note the mixer emits a mic-activity side-channel.
|
||||
- `docs/03-data-model.md`: add `mic_activity.json` to the meeting-dir list (like `voiceprint.wav`).
|
||||
- No new egress; no new default-on setting; `k` threshold lives in code with a calibration comment.
|
||||
|
||||
**Tests.**
|
||||
- Unit: flag-run → span collapse (boundaries, sub-`MIN_SPAN_MS` drop) — pure, table-driven.
|
||||
- Unit: dominance classifier (`mic`, `loop` RMS pairs → You / far / ambiguous), incl. the bleed
|
||||
case (both loud → not You).
|
||||
- Manual: 2-person call on **headphones** → exactly "You" + 1 speaker, near-zero errors; repeat
|
||||
on **speakers** → verify bleed doesn't mislabel far-side as You.
|
||||
|
||||
---
|
||||
|
||||
## 6. Alternatives considered (and why not)
|
||||
|
||||
- **Do nothing / stop at Phase 2.** Legitimate. Phases 0–2 already fix the reported bugs
|
||||
(83-cluster explosion, reprocess collapse, live "You"). Phase 3 is an **accuracy ceiling
|
||||
raise**, not a bug fix. *If 2-person calls are the dominant case and Phase 2's voiceprint "You"
|
||||
tests well in the field, Phase 3 may not clear the cost/benefit bar — measure Phase 2 first.*
|
||||
This is the ponytail-honest recommendation: **verify Phase 2 empirically before building 3.**
|
||||
- **"Mic active → You" (no dominance test).** Simpler, but wrong under speaker bleed (4a). The
|
||||
dominance test is the minimum that survives real rooms.
|
||||
- **Full source separation / AEC.** Much larger, new deps, defeats "minimal + local." The
|
||||
dominance heuristic gets ~90 % of the benefit for ~5 % of the effort.
|
||||
- **Better voiceprint only** (bigger sample, re-match every tick — already done in Phase 2).
|
||||
Still bottlenecked on sherpa's ability to *form* a clean mic cluster in the summed signal;
|
||||
Phase 3's point is to stop relying on that.
|
||||
|
||||
---
|
||||
|
||||
## 7. Open questions for sign-off
|
||||
|
||||
1. **Build it now, or verify Phase 2 first?** Recommendation: run the empirical 2-person
|
||||
re-diarization (outstanding for Phases 0–2) and a Phase-2 live "You" check **before**
|
||||
committing to Phase 3. If Phase 2 accuracy is acceptable, Phase 3 becomes optional.
|
||||
2. **Dominance threshold `k` and mic-VAD floor** — accept a code-constant default with a
|
||||
calibration comment (proposed), or expose a hidden setting? Recommendation: constant first.
|
||||
3. **Far-side signal construction** — zero-out mic-dominant ranges in the loopback lane (simple,
|
||||
preserves timeline) vs. excise-and-concat (shorter audio, shifts timestamps → needs remap).
|
||||
Recommendation: **zero-out**, keeps one timebase.
|
||||
4. **ADR: amend 0005 vs. new ADR-0005a?** Recommendation: amend.
|
||||
5. **Scope of `mic_activity.json`** — persist for reprocess (proposed) or compute live-only and
|
||||
accept that reprocess falls back to voiceprint? Persisting is cheap and keeps reprocess at
|
||||
parity.
|
||||
|
||||
## 8. Rough effort
|
||||
|
||||
Backend-only until the far-side pass; no frontend change (labels already flow via Phase 2). Est.
|
||||
~1–1.5 days: mixer side-channel + accumulator (½ day), span collapse + far-side pass + merge
|
||||
(½ day), persistence + reprocess wiring + docs/tests (½ day). Contained, reversible, behind the
|
||||
existing mic-enabled + retention gates.
|
||||
@@ -0,0 +1,137 @@
|
||||
# Fix plan: speaker diarization accuracy ("You", live labels, 83-speaker explosion)
|
||||
|
||||
Status: planned 2026-07-13 (v0.5.2). Investigation confirmed in code; see memanto memories
|
||||
`a9e5f29e` (diagnosis) and the entry referencing this file (fix order). Execute phases **in
|
||||
order** — each phase is independently shippable and verifiable, and later phases assume
|
||||
earlier ones landed.
|
||||
|
||||
## Symptoms (user report, all reproduced against code)
|
||||
|
||||
1. Live recording shows `S1` for every line; the user's own speech should show "You".
|
||||
2. Post-stop diarization of a 2-person conversation produced **83** speakers.
|
||||
3. Re-transcribing (English Medium) collapsed every segment to speaker "You".
|
||||
|
||||
## Root causes (verified anchors)
|
||||
|
||||
| # | Cause | Anchor |
|
||||
|---|-------|--------|
|
||||
| A | Live segments always carry the pre-diarization `"S1"` placeholder | `src-tauri/src/transcription/mod.rs:222`, emitted at `src-tauri/src/commands.rs:480` |
|
||||
| B | The 15s provisional diarization tick relabels only the backend buffer and emits `diarization://updated`, which **no frontend code listens to** (`src/lib/api.ts` has no listener); live view renders raw labels with no name map (`src/lib/views/TranscriptNotes.svelte:527`) | `src-tauri/src/commands.rs:509-567` |
|
||||
| C | "You" (mic voiceprint match) runs only in `stop_recording`, never live | `src-tauri/src/commands.rs:666-695`, `src-tauri/src/diarization/voiceprint.rs` |
|
||||
| D | `reprocess_transcript` never diarizes: fresh segments all default to `"S1"`, then the meeting's **stale** name map (`S1 → "You"` from the original run) is reused, so the whole transcript renders "You" | `src-tauri/src/commands.rs:1646-1742` |
|
||||
| E | Clustering over the **summed mic+loopback mono** WAV with `num_clusters: -1, threshold: 0.5` over-clusters badly (overlapped speech → mixed embeddings; short pyannote chunks → unstable ERes2Net embeddings) | `src-tauri/src/diarization/mod.rs:68-75` |
|
||||
|
||||
---
|
||||
|
||||
## Phase 0 — Tame the cluster explosion (do first; everything else is useless at 83 clusters)
|
||||
|
||||
Smallest possible change, lands before Phase 1 so re-diarization doesn't reproduce the
|
||||
83-speaker mess.
|
||||
|
||||
1. In `SherpaDiarizer::new` (`src-tauri/src/diarization/mod.rs:73`) raise `threshold` from
|
||||
`0.5` to `0.7` as a code constant with a comment naming the tuning evidence (83 clusters
|
||||
for 2 speakers at 0.5 on mixed mono audio). sherpa fast-clustering semantics: larger
|
||||
threshold → fewer clusters. Do **not** add a settings knob yet.
|
||||
2. In `segment_to_span`'s caller (`SherpaDiarizer::diarize`), drop spans shorter than
|
||||
700 ms before returning — sub-second chunks carry unstable embeddings and only cause
|
||||
label churn in `assign_by_overlap` (which already keeps a segment's prior label when no
|
||||
span overlaps).
|
||||
3. Unit test: spans under the minimum are filtered; existing overlap tests still pass.
|
||||
4. **Verify empirically**: re-run diarization over the user's retained 2-person recording
|
||||
(the 0.5.2 test meeting) and confirm the cluster count lands near 2–4, not 83. If 0.7
|
||||
still over-clusters, try 0.8 before considering per-meeting configurability.
|
||||
|
||||
Acceptance: 2-person retained recording diarizes to ≤4 speakers.
|
||||
|
||||
## Phase 1 — `reprocess_transcript` re-diarizes + persistent mic voiceprint (fixes "everything is You")
|
||||
|
||||
1. **Persist the voiceprint at stop.** In `stop_recording`
|
||||
(`src-tauri/src/commands.rs` after the transcription worker join), when
|
||||
`session.mic_voice_sample` exists **and** `session.retention` is on, write the mic
|
||||
sample as 16 kHz mono `voiceprint.wav` next to `audio.wav` in `meeting_dir(&meeting_id)`.
|
||||
ADR-0009 gate: it is retained audio of the user's voice, so it lives and dies with
|
||||
`audio.wav` — every code path that deletes `audio.wav` (retention off at finalize,
|
||||
delete-recording command) must delete `voiceprint.wav` too. Update
|
||||
`docs/03-data-model.md` (meeting dir file list).
|
||||
2. **Re-diarize on reprocess.** In `reprocess_transcript` (`src-tauri/src/commands.rs:1646`),
|
||||
after `transcribe_file`: build the diarizer via `diarizer_from_installed_models`
|
||||
(inside `spawn_blocking`, same as `stop_recording`), `diarize(&wav_path)`, then
|
||||
`assign(&mut segments, &spans)`. Missing models → skip gracefully (same degradation as
|
||||
live).
|
||||
3. **Rebuild the name map instead of reusing the stale one.** Old labels are meaningless
|
||||
after re-clustering. If `voiceprint.wav` exists, run
|
||||
`voiceprint::match_mic_speaker` against the new spans to get a fresh
|
||||
`You`/`Speaker N` map; otherwise use an empty map (raw `S1…` labels).
|
||||
Pass `speaker_infos_from_segments(&segments, &new_names)` to `finalize_meeting`
|
||||
instead of `meeting.speakers` (`src-tauri/src/commands.rs:1713`), and persist the new
|
||||
names via the store the same way `stop_recording` does. Deliberate policy: user-typed
|
||||
names from the original run are dropped on reprocess because they key to dead labels —
|
||||
note this in the command's doc comment.
|
||||
4. `import_media` reuses this path (`commands.rs:1825` "same batch path"); confirm imported
|
||||
meetings get diarized speakers too (they have no voiceprint — expect raw labels).
|
||||
5. Tests: unit test the fresh-name-map policy (stale map not reused); manual: re-transcribe
|
||||
the 0.5.2 meeting with English Medium → segments show distinct speakers, the user's own
|
||||
lines show "You".
|
||||
|
||||
Acceptance: re-transcription yields per-speaker labels again, with "You" on the mic
|
||||
speaker when a voiceprint exists; never a single-speaker collapse.
|
||||
|
||||
## Phase 2 — Live labels reach the UI, including live "You" (fixes "S1 for everyone")
|
||||
|
||||
Backend (`src-tauri/src/commands.rs`, the 15s tick at 509–567):
|
||||
|
||||
1. After `diarizer.assign` in the tick, **re-emit every committed segment whose speaker
|
||||
changed** via the existing `transcript://segment` event (ids are stable; the frontend
|
||||
store already replaces by id — `src/lib/stores/recording.svelte.ts:49-53`). No new
|
||||
event needed for relabeling.
|
||||
2. In the same tick, run `voiceprint::match_mic_speaker` with
|
||||
`session.mic_voice_sample.samples()` against the fresh spans (clusters re-shuffle every
|
||||
tick, so match every tick; candidate audio is already capped at 10 s per cluster).
|
||||
Merge results into `session.speaker_names` **without overwriting user-set names** —
|
||||
same already-named guard as the post-stop pass (`commands.rs:676-683`). The tick's
|
||||
existing `diarization://updated` emit then carries the "You" display name.
|
||||
3. Frontend: add an `onDiarizationUpdated` listener to `src/lib/api.ts` (payload
|
||||
`{ meetingId, speakers: SpeakerInfo[] }`, already documented in
|
||||
`docs/04-api-contracts.md:188`). Recording store gains a `speakers` state updated by
|
||||
it; live rendering at `TranscriptNotes.svelte:527` passes that list to `speakerName`
|
||||
(the finalized path at :344 already does this).
|
||||
4. Docs: note in `docs/04-api-contracts.md` that `transcript://segment` may re-emit a
|
||||
committed segment with an updated `speaker` (replace-by-id contract).
|
||||
5. Tests: frontend store test — a re-emitted segment with the same id replaces the old
|
||||
one; manual: during a live 2-person call, own speech flips to "You" within ~15–30 s.
|
||||
|
||||
Acceptance: during recording, labels differentiate speakers and the mic speaker shows
|
||||
"You" while talking (within one tick), not just after stop.
|
||||
|
||||
## Phase 3 — Per-stream attribution (accuracy end-game, larger change, needs design sign-off)
|
||||
|
||||
Blind clustering of a summed mono signal is the ceiling on accuracy. WA knows which
|
||||
samples are mic before `spawn_mixer` sums them (`commands.rs:342-371`,
|
||||
`src-tauri/src/audio/mod.rs` MicBridge/mixer). Design sketch — do NOT start without
|
||||
reviewing ADR-0005 and getting sign-off, since it changes the diarization contract:
|
||||
|
||||
- Record a coarse mic-activity timeline during capture (e.g. per-100 ms mic-RMS-dominant
|
||||
flags, negligible memory).
|
||||
- At stop: mic-dominant ranges become "You" spans directly; run sherpa only over the
|
||||
remaining (far-side) ranges to split the *other* participants; merge span lists before
|
||||
`assign_by_overlap`.
|
||||
- Voiceprint match remains as fallback for meetings without the timeline (imports, old
|
||||
recordings).
|
||||
- Update ADR-0005 (or add a new ADR) + `docs/02-architecture.md` data flow.
|
||||
|
||||
Acceptance: 2-person call yields exactly "You" + 1 speaker with near-zero attribution
|
||||
errors on non-overlapping speech; overlapping speech attributes to the dominant stream.
|
||||
|
||||
---
|
||||
|
||||
## Cross-cutting rules for the executing agent
|
||||
|
||||
- CLAUDE.md applies in full: `cargo fmt` + `cargo clippy -- -D warnings`, no
|
||||
`unwrap()`/`expect()` on user-reachable paths, conventional commits referencing
|
||||
FR-SPK-*, commit each file right after finishing it (one commit per file), docs updated
|
||||
in the same change when contracts move.
|
||||
- Memory discipline: `memanto agent activate whispassist` first; `memanto remember` every
|
||||
decision/tuning result (especially the empirical threshold from Phase 0) with full
|
||||
metadata; record phase completion so the next agent knows where to resume.
|
||||
- No new egress, no new settings defaults ON; `voiceprint.wav` is local retained audio
|
||||
under the existing ADR-0009 consent/retention gate.
|
||||
@@ -0,0 +1,150 @@
|
||||
# Plan: dual-channel capture (mic = left, loopback = right) — FR-SPK / FR-CAP
|
||||
|
||||
Status: **implemented** 2026-07-14 (all 3 sign-off answers: DB column; retire MicActivity with a
|
||||
code note; export → dual-mono). Migration 0009 + `write_wav_split` + `attribute_split` +
|
||||
`fold_wav` + ADR-0005 Phase 3.5. Supersedes the summed-mono + masked-diarize approach
|
||||
in [`2026-07-13-diarization-phase3-per-stream-attribution.md`](./2026-07-13-diarization-phase3-per-stream-attribution.md).
|
||||
The proposed **ADR-0005 amendment** is at the bottom — apply it on approval.
|
||||
|
||||
## Motivation
|
||||
|
||||
Every diarization problem this month traces to one root: **`audio.wav` is a summed mono mix** of
|
||||
mic + loopback, so the two sources are irrecoverable after the fact. Phase 3 works around that by
|
||||
(a) recording a separate `mic_activity.json` "You" timeline and (b) diarizing the mix with the You
|
||||
ranges masked out. It works at stop but is fragile on **reprocess** — the persisted timeline has to
|
||||
re-align against freshly-transcribed segments over a mono mix, and it currently collapses to a
|
||||
single "Speaker 2". The timelines are being written correctly (verified: real `you_spans`), so the
|
||||
failure is the masked-mono re-alignment itself, not the capture.
|
||||
|
||||
**Fix the root, not the instance:** keep both sources *physically separate in the recording* by
|
||||
recording **stereo — left channel = microphone ("You"), right channel = system/loopback
|
||||
("Speaker")**. The separation then lives in the file forever; reprocess and diarization always have
|
||||
both streams; no sidecar timeline, no masking, no summed-mono clustering.
|
||||
|
||||
## Key facts that make this cheap
|
||||
|
||||
- `audio.wav` is **already 2-channel** on a normal (stereo) render endpoint — today we *sum the mic
|
||||
into both channels*. Writing mic→L and loopback(downmixed)→R is the **same channel count, same
|
||||
bytes** — no size increase in the common case.
|
||||
- Transcription already downmixes to mono: `read_wav_mono_16k` averaging L+R of a split file yields
|
||||
`(mic + loopback)/2` = the same summed signal it gets today, so **transcription is unchanged**.
|
||||
- The mic↔loopback alignment machinery (`MicBridge`, the drain fix, the capture-silence fix) carries
|
||||
over unchanged — we just route the aligned samples to two channels instead of summing them.
|
||||
|
||||
## Design
|
||||
|
||||
### 1. Capture write path (`audio`)
|
||||
- Force the recording spec to **2ch, 16-bit, native rate** when the mic is enabled ("split"
|
||||
layout). Loopback-only (mic off) keeps today's behavior.
|
||||
- New writer (replacing `write_wav_bytes`'s summing for split recordings): per frame,
|
||||
**L = mic sample** (0 when the mic underran), **R = downmix(loopback channels)**.
|
||||
- Idle path (loopback silent): **L = drained mic, R = 0** — same silence-synthesis + device-paced
|
||||
drain we just fixed, now writing to L instead of summing.
|
||||
|
||||
### 2. Layout versioning (distinguish old "summed" from new "split")
|
||||
Content alone can't tell a summed-stereo file from a split-stereo one, so mark it explicitly:
|
||||
- Add `audio_layout TEXT` to the `meetings` row: `'summed'` (default / NULL for all existing rows
|
||||
and imports) or `'split'` (new mic-enabled recordings).
|
||||
- Include it in the **`MeetingBundle` manifest** so exported recordings stay interpretable on
|
||||
another machine.
|
||||
- Playback and diarization branch on it; old recordings keep every current code path untouched.
|
||||
|
||||
### 3. Read helpers (`audio`)
|
||||
- `read_wav_mono_16k` — **unchanged** (downmix all channels → mono 16k). Used for transcription of
|
||||
both layouts; on a split file it returns the summed signal, exactly as wanted.
|
||||
- New `read_wav_channel_16k(path, channel)` — extract one channel → mono 16k. Diarization reads the
|
||||
**right** channel (far side); "You" detection reads the **left** (mic).
|
||||
|
||||
### 4. Diarization / attribution (`commands`) — *simpler* than Phase 3
|
||||
For a **split** recording, one path replaces the timeline + masking:
|
||||
```
|
||||
far_spans = diarizer.diarize_samples( read_wav_channel_16k(wav, RIGHT) ) // clean far side
|
||||
you_spans = vad_spans( read_wav_channel_16k(wav, LEFT) ) // energy-windowed mic
|
||||
merge + assign_by_overlap + build_name_map("You") // You + Speaker N
|
||||
```
|
||||
- The mic is **never** in the clustering input — structurally, not procedurally.
|
||||
- `vad_spans` is the `MicActivity` windowing logic (energy floor + 100 ms windows) applied to the L
|
||||
channel from the file — so **stop and reprocess share one path** and produce identical results.
|
||||
- **This deletes `MicActivity`, `mic_activity.json` (persist + read + delete), and `mask_ranges`.**
|
||||
Net: less code than the current Phase 3.
|
||||
- Mic-off (`'summed'`) recordings keep the whole-signal pass + voiceprint fallback.
|
||||
|
||||
### 5. Playback (`commands::serve_recording`)
|
||||
- A split file plays mic-in-left / computer-in-right — wrong for a listener. On playback, **downmix
|
||||
L+R → mono** (dual-mono) so both sources come out both ears.
|
||||
- `serve_recording` already reads+decrypts the whole file into memory; add a downmix-to-mono step
|
||||
for `'split'` layout and serve that mono WAV (range/seek computed over the transcoded bytes).
|
||||
`'summed'` files stream as today.
|
||||
|
||||
### 6. Migration & portability
|
||||
- **Existing recordings** (`'summed'`): every current path unchanged. Their reprocess falls back to
|
||||
the voiceprint match (acceptable — they predate split). We may retire the `MicActivity`/mask code
|
||||
once split is default; the handful of recent test recordings lose Phase 3 on reprocess, which is
|
||||
fine.
|
||||
- **Bundle export**: export **downmixes the canonical split `audio.wav` to mono** so a shared file
|
||||
plays normally in any player; the working file stays split for WA's own re-processing.
|
||||
|
||||
## What this removes / simplifies
|
||||
|
||||
| Removed | Replaced by |
|
||||
|---|---|
|
||||
| `MicActivity` struct + capture wiring | L-channel VAD read from the file |
|
||||
| `mic_activity.json` (write/read/delete) | nothing — recompute from L channel |
|
||||
| `mask_ranges` + masked diarize | diarize the R channel directly |
|
||||
| stop vs reprocess divergence | one shared split-attribution path |
|
||||
|
||||
The capture-silence fix, the device-paced `drain`, `diarize_samples`, and the `build_name_map`
|
||||
naming all **stay**.
|
||||
|
||||
## Costs / risks (none fatal)
|
||||
- **New code:** split writer, `audio_layout` column + migration + manifest field, channel read
|
||||
helper, playback downmix, `vad_spans`, and the split-vs-summed branch. Bigger than a patch, but
|
||||
it *retires* comparable Phase 3 code.
|
||||
- **Loopback stereo image lost** (R = loopback downmixed to mono) — irrelevant for speech.
|
||||
- **Mono render endpoint** (rare): 1ch→2ch doubles that recording's size.
|
||||
- **Raw file plays split** in a third-party player until exported (export downmixes) — minor.
|
||||
- **Clock drift** between mic/loopback now shows as tiny L/R desync instead of dropped summed
|
||||
samples — harmless (diarization tolerates it; playback sums it away).
|
||||
|
||||
## Open questions for sign-off
|
||||
1. Layout marker: **DB column** (proposed) vs. a self-describing WAV chunk? DB column is simpler and
|
||||
travels via the bundle manifest; recommend it.
|
||||
2. Retire `MicActivity`/`mask_ranges` outright, or keep them for existing `'summed'` recordings'
|
||||
reprocess? Recommend **retire** (summed → voiceprint fallback) to avoid two live systems.
|
||||
3. Export downmix to mono — agree that's the right default for shared/synced copies?
|
||||
|
||||
## Implementation order
|
||||
1. `audio_layout` column + `MeetingBundle` field + read on `get_meeting` (no behavior yet).
|
||||
2. Split writer + force 2ch when mic on; set `audio_layout='split'`. (`audio` + `start_recording`.)
|
||||
3. `read_wav_channel_16k` + `vad_spans`; split-attribution path in stop + reprocess.
|
||||
4. Playback downmix for split; bundle-export downmix.
|
||||
5. Delete `MicActivity` / `mic_activity.json` / `mask_ranges`; update tests.
|
||||
6. Apply the ADR-0005 amendment + update `docs/02`/`03`.
|
||||
|
||||
---
|
||||
|
||||
## Proposed ADR-0005 amendment (apply on approval)
|
||||
|
||||
> ### Phase 3.5 refinement — dual-channel capture (FR-SPK/FR-CAP, 2026-07-14)
|
||||
>
|
||||
> The Phase 3 per-stream attribution (above) reconstructs the mic/far-side split *after the fact*
|
||||
> from a summed-mono `audio.wav` plus a `mic_activity.json` timeline, then masks the mic ranges
|
||||
> before clustering. This is reliable at stop but fragile on reprocess (re-aligning a sidecar
|
||||
> timeline against a mono mix), and it carries a parallel persistence path.
|
||||
>
|
||||
> **Decision:** when the microphone is enabled, record `audio.wav` as **stereo with the streams
|
||||
> separated — left = microphone ("You"), right = system/loopback ("Speaker")** — instead of summing
|
||||
> them. The separation is then intrinsic to the recording:
|
||||
> - Diarization runs on the **right channel only** → `Speaker N`; "You" comes from **left-channel**
|
||||
> voice activity. The mic is never clustered, by construction.
|
||||
> - Reprocess recomputes both from the file — no sidecar timeline, no masking.
|
||||
> - Transcription still downmixes (L+R) to the same summed mono it uses today; playback and bundle
|
||||
> export downmix to mono so shared/played audio is normal.
|
||||
> - A `meetings.audio_layout` flag (`summed` | `split`, in the bundle manifest for portability)
|
||||
> distinguishes recordings; existing `summed` recordings keep the original blind-clustering +
|
||||
> voiceprint path. On a stereo render endpoint this is **size-neutral** (the file was already 2ch
|
||||
> with the mic summed into both).
|
||||
>
|
||||
> This **supersedes** `MicActivity` / `mic_activity.json` / masked diarization from Phase 3; the
|
||||
> capture-silence handling, `diarize_samples`, and the `build_name_map` naming are retained. Segment
|
||||
> IDs and the names-in-DB rule are unchanged.
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "whispassist",
|
||||
"private": true,
|
||||
"version": "0.5.2",
|
||||
"version": "0.6.0",
|
||||
"type": "module",
|
||||
"description": "Privacy-first, fully local Windows meeting assistant.",
|
||||
"license": "MIT OR Apache-2.0",
|
||||
|
||||
Generated
+1
-1
@@ -6043,7 +6043,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "whispassist"
|
||||
version = "0.5.2"
|
||||
version = "0.6.0"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "whispassist"
|
||||
version = "0.5.2"
|
||||
version = "0.6.0"
|
||||
description = "Privacy-first, fully local Windows meeting assistant"
|
||||
authors = ["WhispAssist contributors"]
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
-- WhispAssist dual-channel capture (FR-SPK/FR-CAP). Forward-only migration.
|
||||
-- Mirrors docs/03-data-model.md.
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
-- How audio.wav stores its channels:
|
||||
-- NULL / 'summed' — mic summed into every loopback channel (all pre-2026-07-14
|
||||
-- recordings and every import): one mixed mono-equivalent.
|
||||
-- 'split' — stereo, left = microphone ("You"), right = system/loopback
|
||||
-- ("Speaker"), kept separate so diarization runs on the far
|
||||
-- side alone. New mic-enabled recordings.
|
||||
-- Diarization, playback (downmix), and bundle export branch on this.
|
||||
ALTER TABLE meetings ADD COLUMN audio_layout TEXT;
|
||||
+314
-26
@@ -201,16 +201,37 @@ impl MicBridge {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Loopback idle path: take everything buffered at once, device-paced. The
|
||||
/// idle writer must NOT meter the mic by wall-clock — that lets the buffer
|
||||
/// back up to `cap` and drop the oldest samples, which sounds like the audio
|
||||
/// skipping/fast-forwarding late in a long silent stretch. Draining fully
|
||||
/// keeps the mic flowing at its own clock.
|
||||
fn drain(&self) -> Vec<f32> {
|
||||
self.buf
|
||||
.lock()
|
||||
.map(|mut buf| buf.drain(..).collect())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Below this per-chunk RMS the mic is treated as silence/room noise and not
|
||||
/// collected into a voiceprint. Without it the voiceprint was just the first
|
||||
/// few seconds of the recording regardless of whether the user had spoken —
|
||||
/// so a user silent for the first 30s got a noise voiceprint that spuriously
|
||||
/// matched a cluster and mislabeled everyone "You". ponytail: fixed floor for
|
||||
/// normalized [-1,1] speech; promote to a calibration knob if real mics vary.
|
||||
const VOICE_ENERGY_FLOOR: f32 = 0.01;
|
||||
|
||||
/// A one-shot, bounded capture of raw mic-only audio (16kHz mono, same format
|
||||
/// the transcriber and diarizer both use) taken early in a recording — enough
|
||||
/// to compute a voiceprint that identifies which diarized speaker cluster is
|
||||
/// the mic (so it can be labeled "You" instead of a clustered "S1"/"S2"; see
|
||||
/// `diarization::voiceprint`). Unlike `MicBridge`, this is filled once and
|
||||
/// never drained: the first `cap` samples are kept and everything after is
|
||||
/// dropped, since a voiceprint only needs a few seconds of real speech, not
|
||||
/// the whole meeting.
|
||||
/// the transcriber and diarizer both use) taken from the user's early *speech*
|
||||
/// — enough to compute a voiceprint that identifies which diarized speaker
|
||||
/// cluster is the mic (so it can be labeled "You" instead of a clustered
|
||||
/// "S1"/"S2"; see `diarization::voiceprint`). Unlike `MicBridge`, this is filled
|
||||
/// once and never drained: the first `cap` samples of speech-level audio are
|
||||
/// kept and everything after is dropped, since a voiceprint only needs a few
|
||||
/// seconds of real speech, not the whole meeting. Silence is skipped so the
|
||||
/// window tracks when the user actually talks, not the wall clock.
|
||||
pub struct VoiceSample {
|
||||
cap: usize,
|
||||
buf: Mutex<Vec<f32>>,
|
||||
@@ -225,6 +246,18 @@ impl VoiceSample {
|
||||
}
|
||||
|
||||
fn push(&self, samples: &[f32]) {
|
||||
// Skip silence/room noise so the voiceprint is the user's voice, not
|
||||
// whatever the mic picked up before they first spoke (see
|
||||
// VOICE_ENERGY_FLOOR). RMS inline so VoiceSample stays independent of
|
||||
// the `audio`-gated level helper.
|
||||
let rms = if samples.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
(samples.iter().map(|s| s * s).sum::<f32>() / samples.len() as f32).sqrt()
|
||||
};
|
||||
if rms < VOICE_ENERGY_FLOOR {
|
||||
return;
|
||||
}
|
||||
if let Ok(mut buf) = self.buf.lock() {
|
||||
if buf.len() < self.cap {
|
||||
buf.extend_from_slice(samples);
|
||||
@@ -240,6 +273,12 @@ impl VoiceSample {
|
||||
}
|
||||
}
|
||||
|
||||
// MicActivity (the live mic-speech timeline) was retired 2026-07-14 with the
|
||||
// masked-mono diarization it fed — the split (mic-left/loopback-right) layout
|
||||
// makes the mic/far-side separation intrinsic to audio.wav, so "You" is
|
||||
// recomputed from the left channel via `vad_spans`. See the retirement note in
|
||||
// commands.rs (attribute_split).
|
||||
|
||||
/// Number of audio frames in a raw WASAPI byte buffer of the given format.
|
||||
#[cfg(feature = "audio")]
|
||||
fn frame_count(bytes: &[u8], format: &WaveFormat) -> usize {
|
||||
@@ -273,6 +312,7 @@ impl WasapiCapture {
|
||||
event_sink: EventSink,
|
||||
bridge: Option<Arc<MicBridge>>,
|
||||
voice_sample: Option<Arc<VoiceSample>>,
|
||||
split: bool,
|
||||
emit_level: bool,
|
||||
) -> Result<CaptureHandle, AudioError> {
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
@@ -295,6 +335,7 @@ impl WasapiCapture {
|
||||
&paused_th,
|
||||
bridge.as_ref(),
|
||||
voice_sample.as_ref(),
|
||||
split,
|
||||
emit_level,
|
||||
)
|
||||
})
|
||||
@@ -327,7 +368,8 @@ impl WasapiCapture {
|
||||
event_sink,
|
||||
Some(bridge),
|
||||
None,
|
||||
true,
|
||||
true, // split: mic-enabled loopback records mic-left / loopback-right
|
||||
true, // emit_level
|
||||
)
|
||||
}
|
||||
|
||||
@@ -352,7 +394,8 @@ impl WasapiCapture {
|
||||
event_sink,
|
||||
Some(bridge),
|
||||
voice_sample,
|
||||
true,
|
||||
false, // split: mic stream has no writer
|
||||
true, // emit_level
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -375,7 +418,8 @@ impl AudioCapture for WasapiCapture {
|
||||
event_sink,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
false, // split: loopback-only / mic stream — no split writer
|
||||
true, // emit_level
|
||||
)
|
||||
}
|
||||
|
||||
@@ -394,7 +438,8 @@ impl AudioCapture for WasapiCapture {
|
||||
event_sink,
|
||||
None,
|
||||
None,
|
||||
true,
|
||||
false, // split: loopback-only / mic stream — no split writer
|
||||
true, // emit_level
|
||||
)
|
||||
}
|
||||
|
||||
@@ -540,6 +585,18 @@ fn audio_level(mono: &[f32], mic: bool) -> AudioLevel {
|
||||
/// without flooding the frontend with an event per ~10ms WASAPI callback.
|
||||
const LEVEL_EMIT_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
/// When the loopback (render) endpoint is idle, WASAPI delivers no packets, so
|
||||
/// the recording would stall and the bridged mic — the user talking over system
|
||||
/// silence — would be lost as the MicBridge overflows. The loopback loop then
|
||||
/// synthesizes silence for the elapsed gap and folds the mic in, keeping
|
||||
/// `audio.wav` continuous and complete (FR-CAP-7). MIN gates on a *sustained*
|
||||
/// gap so brief inter-packet stalls during active playback don't punch silence
|
||||
/// into the loopback (audible clicks); it stays well under the MicBridge's
|
||||
/// ~0.5s cap so the mic can't back up and drop samples between drains. MAX caps
|
||||
/// the silence padding written in one chunk.
|
||||
const SILENCE_TICK_MIN: Duration = Duration::from_millis(150);
|
||||
const SILENCE_TICK_MAX: Duration = Duration::from_millis(500);
|
||||
|
||||
/// Runs on a dedicated OS thread for the lifetime of a `CaptureHandle`. Owns the
|
||||
/// WASAPI client and the WAV writer; exits (and finalizes the WAV) once `running`
|
||||
/// is cleared.
|
||||
@@ -555,6 +612,10 @@ fn capture_loop(
|
||||
paused: &AtomicBool,
|
||||
bridge: Option<&Arc<MicBridge>>,
|
||||
voice_sample: Option<&Arc<VoiceSample>>,
|
||||
// FR-SPK: when true, the loopback WAV is stereo L=mic / R=loopback (the mic
|
||||
// is a separate track, not summed in) so diarization can run on the far side
|
||||
// alone. Only meaningful for the loopback writer.
|
||||
split: bool,
|
||||
emit_level: bool,
|
||||
) -> Result<CaptureSummary, AudioError> {
|
||||
wasapi::initialize_mta()
|
||||
@@ -566,14 +627,18 @@ fn capture_loop(
|
||||
// UI can overlay them — but only the loopback stream writes a WAV.
|
||||
let mut session = open_capture_session(&direction, device_id)?;
|
||||
// Loopback publishes its rate so the mic knows what to resample to before
|
||||
// pushing into the shared bridge (mic-into-recording, FR-CAP-7).
|
||||
// pushing into the shared bridge (mic-into-recording, FR-CAP-7), and so the
|
||||
// Phase 3 mic-activity timeline maps frame positions to windows.
|
||||
if is_loopback {
|
||||
if let Some(b) = bridge {
|
||||
b.rate
|
||||
.store(session.format.get_samplespersec(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
let spec = wav_spec_for(&session.format)?;
|
||||
let mut spec = wav_spec_for(&session.format)?;
|
||||
if split && is_loopback {
|
||||
spec.channels = 2; // L = mic, R = loopback (downmixed)
|
||||
}
|
||||
let mut writer = match wav_path {
|
||||
Some(path) => Some(WavWriter::create(path, spec).map_err(|e| {
|
||||
AudioError::Capture(format!("could not create {}: {e}", path.display()))
|
||||
@@ -593,6 +658,9 @@ fn capture_loop(
|
||||
let mut queue: std::collections::VecDeque<u8> = std::collections::VecDeque::new();
|
||||
let mut frames_written: u64 = 0;
|
||||
let mut last_level_emit = Instant::now() - LEVEL_EMIT_INTERVAL;
|
||||
// Wall-clock of the last WAV write (real or synthesized), so the loopback
|
||||
// loop can fill idle gaps with silence+mic instead of stalling.
|
||||
let mut last_write = Instant::now();
|
||||
|
||||
while running.load(Ordering::Relaxed) {
|
||||
// Short timeout so we periodically re-check `running` even with no data.
|
||||
@@ -636,8 +704,43 @@ fn capture_loop(
|
||||
continue;
|
||||
}
|
||||
if queue.is_empty() {
|
||||
// Loopback idle (nothing playing): keep audio.wav advancing so the
|
||||
// bridged mic (user talking over system silence) is preserved
|
||||
// instead of lost to MicBridge overflow. Only after a *sustained*
|
||||
// gap, so brief inter-packet stalls during active playback don't
|
||||
// punch silence into the loopback (clicks). Loopback + mic-enabled
|
||||
// only; the mic capture stream has no writer.
|
||||
if is_loopback && !paused.load(Ordering::Relaxed) {
|
||||
if let (Some(w), Some(b)) = (writer.as_mut(), bridge) {
|
||||
let elapsed = last_write.elapsed();
|
||||
if elapsed >= SILENCE_TICK_MIN {
|
||||
// Drain the whole mic buffer (device-paced). Metering the
|
||||
// pull by wall-clock let the buffer back up to its cap and
|
||||
// drop the oldest samples — the audio skipped/fast-
|
||||
// forwarded late in a long gap. Draining fully avoids that.
|
||||
let mic = b.drain();
|
||||
let rate = session.format.get_samplespersec() as usize;
|
||||
// Advance the timeline by the wall-clock gap (capped), but
|
||||
// never fewer frames than the mic we must write.
|
||||
let gap_frames = (elapsed.min(SILENCE_TICK_MAX).as_micros() as usize
|
||||
* rate
|
||||
/ 1_000_000)
|
||||
.max(mic.len());
|
||||
let bpf = session.format.get_nchannels() as usize
|
||||
* (session.format.get_bitspersample() as usize / 8);
|
||||
let silence = vec![0u8; gap_frames * bpf];
|
||||
frames_written += if split {
|
||||
write_wav_split(w, &silence, &session.format, &mic)?
|
||||
} else {
|
||||
write_wav_bytes(w, &silence, &session.format, &mic)?
|
||||
};
|
||||
last_write = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
last_write = Instant::now();
|
||||
let bytes: Vec<u8> = queue.drain(..).collect();
|
||||
|
||||
// Must keep pulling WASAPI buffers even while paused (required to avoid
|
||||
@@ -653,7 +756,11 @@ fn capture_loop(
|
||||
Some(b) if is_loopback => b.pull(frame_count(&bytes, &session.format)),
|
||||
_ => Vec::new(),
|
||||
};
|
||||
frames_written += write_wav_bytes(w, &bytes, &session.format, &mic)?;
|
||||
frames_written += if split {
|
||||
write_wav_split(w, &bytes, &session.format, &mic)?
|
||||
} else {
|
||||
write_wav_bytes(w, &bytes, &session.format, &mic)?
|
||||
};
|
||||
}
|
||||
let mono = decode_mono_f32(&bytes, &session.format)?;
|
||||
|
||||
@@ -783,6 +890,53 @@ fn f32_to_i16(v: f32) -> i16 {
|
||||
(v.clamp(-1.0, 1.0) * i16::MAX as f32) as i16
|
||||
}
|
||||
|
||||
/// Split layout (FR-SPK): write frames as **stereo, L = mic, R = loopback**
|
||||
/// (downmixed to mono), so the two sources stay separate in `audio.wav` and
|
||||
/// diarization can run on the far side alone. `loopback_bytes` is the raw WASAPI
|
||||
/// mix buffer (all-zero for a synthesized-silence chunk); `mic` may be shorter
|
||||
/// than the frame count (rest silent) or empty (mic underran). Returns frames.
|
||||
fn write_wav_split(
|
||||
writer: &mut WavWriter<BufWriter<File>>,
|
||||
loopback_bytes: &[u8],
|
||||
format: &WaveFormat,
|
||||
mic: &[f32],
|
||||
) -> Result<u64, AudioError> {
|
||||
let sample_type = format
|
||||
.get_subformat()
|
||||
.map_err(|e| AudioError::Device(format!("unrecognized mix format: {e}")))?;
|
||||
let channels = format.get_nchannels() as usize;
|
||||
if channels == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let (bytes_per_sample, decode): (usize, fn(&[u8]) -> f32) = match (sample_type, format.get_bitspersample()) {
|
||||
(SampleType::Float, 32) => (4, |c| f32::from_le_bytes(c.try_into().unwrap())),
|
||||
(SampleType::Int, 16) => (2, |c| {
|
||||
i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32
|
||||
}),
|
||||
(st, bits) => {
|
||||
return Err(AudioError::Device(format!(
|
||||
"unsupported capture format: {st} {bits}-bit"
|
||||
)))
|
||||
}
|
||||
};
|
||||
let mut frames = 0u64;
|
||||
for frame in loopback_bytes.chunks_exact(bytes_per_sample * channels) {
|
||||
// R = loopback downmixed to mono.
|
||||
let loop_mono =
|
||||
frame.chunks_exact(bytes_per_sample).map(decode).sum::<f32>() / channels as f32;
|
||||
// L = mic for this frame (silent if the mic underran).
|
||||
let mic_s = mic.get(frames as usize).copied().unwrap_or(0.0);
|
||||
writer
|
||||
.write_sample(f32_to_i16(mic_s))
|
||||
.map_err(|e| AudioError::Capture(format!("wav write: {e}")))?;
|
||||
writer
|
||||
.write_sample(f32_to_i16(loop_mono))
|
||||
.map_err(|e| AudioError::Capture(format!("wav write: {e}")))?;
|
||||
frames += 1;
|
||||
}
|
||||
Ok(frames)
|
||||
}
|
||||
|
||||
/// Downmix raw WASAPI capture bytes to mono `f32` in `[-1.0, 1.0]`, at the
|
||||
/// device's native sample rate (resampling to 16kHz happens separately).
|
||||
fn decode_mono_f32(bytes: &[u8], format: &WaveFormat) -> Result<Vec<f32>, AudioError> {
|
||||
@@ -897,6 +1051,22 @@ impl Resampler {
|
||||
/// through hound's size-aware iterator; hound is only used to parse the
|
||||
/// format/`fmt ` chunk.
|
||||
pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
|
||||
read_wav_16k(path, None)
|
||||
}
|
||||
|
||||
/// Like [`read_wav_mono_16k`] but extracts a single channel instead of
|
||||
/// downmixing — used by the Phase 3.5 split path to diarize the right
|
||||
/// (loopback) channel and VAD the left (mic) channel (FR-SPK). `channel` is
|
||||
/// clamped to the file's last channel, so it's safe on a mono/summed file.
|
||||
pub fn read_wav_channel_16k(path: &Path, channel: usize) -> Result<Vec<f32>, AudioError> {
|
||||
read_wav_16k(path, Some(channel))
|
||||
}
|
||||
|
||||
/// Shared reader: decrypt (T8.8), reduce each frame to one f32 — averaging all
|
||||
/// channels (`channel = None`) or picking one (`channel = Some`) — resample to
|
||||
/// 16kHz. Reads the `data` payload as raw bytes (an unfinalized WAV's size
|
||||
/// fields read back as 0), so it works on a mid-write / crash-left file (FR-REL-1).
|
||||
fn read_wav_16k(path: &Path, channel: Option<usize>) -> Result<Vec<f32>, AudioError> {
|
||||
let raw = std::fs::read(path).map_err(|e| AudioError::Capture(format!("read wav: {e}")))?;
|
||||
// Decrypt if the recording was sealed at rest (T8.8); plaintext passes through.
|
||||
let file =
|
||||
@@ -906,6 +1076,8 @@ pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
|
||||
.map_err(|e| AudioError::Capture(format!("open wav: {e}")))?
|
||||
.spec();
|
||||
let channels = spec.channels.max(1) as usize;
|
||||
// Which channel index to pick (clamped), or None to downmix.
|
||||
let pick = channel.map(|c| c.min(channels - 1));
|
||||
|
||||
let data_marker = file
|
||||
.windows(4)
|
||||
@@ -917,22 +1089,14 @@ pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
|
||||
let mono: Vec<f32> = match (spec.sample_format, spec.bits_per_sample) {
|
||||
(SampleFormat::Float, 32) => bytes
|
||||
.chunks_exact(4 * channels)
|
||||
.map(|frame| {
|
||||
frame
|
||||
.chunks_exact(4)
|
||||
.map(|c| f32::from_le_bytes(c.try_into().unwrap()))
|
||||
.sum::<f32>()
|
||||
/ channels as f32
|
||||
})
|
||||
.map(|frame| reduce_frame(frame, 4, pick, channels, |c| f32::from_le_bytes(c.try_into().unwrap())))
|
||||
.collect(),
|
||||
(SampleFormat::Int, 16) => bytes
|
||||
.chunks_exact(2 * channels)
|
||||
.map(|frame| {
|
||||
frame
|
||||
.chunks_exact(2)
|
||||
.map(|c| i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32)
|
||||
.sum::<f32>()
|
||||
/ channels as f32
|
||||
reduce_frame(frame, 2, pick, channels, |c| {
|
||||
i16::from_le_bytes(c.try_into().unwrap()) as f32 / i16::MAX as f32
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
(fmt, bits) => {
|
||||
@@ -946,6 +1110,78 @@ pub fn read_wav_mono_16k(path: &Path) -> Result<Vec<f32>, AudioError> {
|
||||
Ok(resampler.process(&mono))
|
||||
}
|
||||
|
||||
/// Voice-activity "You" spans (FR-SPK): 100ms windows of 16kHz mono audio above
|
||||
/// [`VOICE_ENERGY_FLOOR`], merged into `(start_ms, end_ms)` ranges. Run on the
|
||||
/// left (mic) channel of a split recording to attribute segments to "You" —
|
||||
/// replaces the old live `MicActivity` timeline (recomputed from the file, so
|
||||
/// stop and reprocess agree).
|
||||
pub fn vad_spans(samples: &[f32]) -> Vec<(u64, u64)> {
|
||||
const WINDOW_MS: u64 = 100;
|
||||
let win = (TARGET_SAMPLE_RATE as u64 * WINDOW_MS / 1000) as usize; // 1600 @16kHz
|
||||
if win == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut spans = Vec::new();
|
||||
let mut start: Option<u64> = None;
|
||||
let mut w = 0u64;
|
||||
let mut i = 0usize;
|
||||
while i < samples.len() {
|
||||
let chunk = &samples[i..(i + win).min(samples.len())];
|
||||
let rms = (chunk.iter().map(|s| s * s).sum::<f32>() / chunk.len().max(1) as f32).sqrt();
|
||||
match (rms >= VOICE_ENERGY_FLOOR, start) {
|
||||
(true, None) => start = Some(w * WINDOW_MS),
|
||||
(false, Some(s)) => {
|
||||
spans.push((s, w * WINDOW_MS));
|
||||
start = None;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
i += win;
|
||||
w += 1;
|
||||
}
|
||||
if let Some(s) = start {
|
||||
spans.push((s, w * WINDOW_MS));
|
||||
}
|
||||
spans
|
||||
}
|
||||
|
||||
/// Reduce one interleaved frame to a single f32: pick channel `pick` or average
|
||||
/// all `channels`. `width` is bytes-per-sample; `decode` turns those bytes → f32.
|
||||
fn reduce_frame(
|
||||
frame: &[u8],
|
||||
width: usize,
|
||||
pick: Option<usize>,
|
||||
channels: usize,
|
||||
decode: impl Fn(&[u8]) -> f32,
|
||||
) -> f32 {
|
||||
match pick {
|
||||
Some(c) => decode(&frame[c * width..c * width + width]),
|
||||
None => frame.chunks_exact(width).map(decode).sum::<f32>() / channels as f32,
|
||||
}
|
||||
}
|
||||
|
||||
/// Write mono 16kHz `f32` samples as a Float32 WAV — the format
|
||||
/// [`read_wav_mono_16k`] reads back. Used to persist the mic voiceprint sample
|
||||
/// next to a retained recording so it can be re-matched on reprocess (FR-SPK).
|
||||
pub fn write_wav_mono_16k(path: &Path, samples: &[f32]) -> Result<(), AudioError> {
|
||||
let spec = WavSpec {
|
||||
channels: 1,
|
||||
sample_rate: 16_000,
|
||||
bits_per_sample: 32,
|
||||
sample_format: SampleFormat::Float,
|
||||
};
|
||||
let mut writer =
|
||||
WavWriter::create(path, spec).map_err(|e| AudioError::Capture(format!("write wav: {e}")))?;
|
||||
for &s in samples {
|
||||
writer
|
||||
.write_sample(s)
|
||||
.map_err(|e| AudioError::Capture(format!("write wav: {e}")))?;
|
||||
}
|
||||
writer
|
||||
.finalize()
|
||||
.map_err(|e| AudioError::Capture(format!("write wav: {e}")))
|
||||
}
|
||||
|
||||
/// Ceiling on how far the still-flowing stream may run ahead of a stalled
|
||||
/// partner before the mixer forwards it alone — so a muted/dead microphone (or a
|
||||
/// silent system output) can't hold up the live transcript. 0.5s @ 16kHz.
|
||||
@@ -1051,6 +1287,31 @@ fn mixer_loop(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn vad_spans_ignores_silence() {
|
||||
// 200ms of 16kHz silence → no "You" spans.
|
||||
assert!(vad_spans(&[0.0; 3_200]).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vad_spans_collapses_speech_into_spans_with_gaps() {
|
||||
// 16kHz: 1600 samples per 100ms window. Speech in windows 0,1 (0-200ms)
|
||||
// and 4,5 (400-600ms), silence between.
|
||||
let mut samples = vec![0.0f32; 9_600]; // 600ms
|
||||
samples[0..3_200].fill(0.2); // windows 0,1
|
||||
samples[6_400..9_600].fill(0.2); // windows 4,5
|
||||
assert_eq!(vad_spans(&samples), vec![(0, 200), (400, 600)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn voice_sample_skips_silence_and_keeps_speech() {
|
||||
let vs = VoiceSample::new(16_000);
|
||||
vs.push(&[0.0; 4_000]); // silence: below the energy floor
|
||||
assert!(vs.samples().is_empty(), "silence must not fill the voiceprint");
|
||||
vs.push(&[0.2; 4_000]); // speech-level audio
|
||||
assert_eq!(vs.samples().len(), 4_000, "speech-level audio is collected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixer_sums_aligned_streams_and_keeps_the_unmatched_tail() {
|
||||
let mut m = Mixer::new();
|
||||
@@ -1127,6 +1388,33 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_wav_split_puts_mic_left_and_loopback_right() {
|
||||
let dir = std::env::temp_dir().join(format!("wa-test-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("audio.wav");
|
||||
// 2ch float loopback input; split output is always 2ch 16-bit.
|
||||
let format = WaveFormat::new(32, 32, &SampleType::Float, 48_000, 2, None);
|
||||
let spec = WavSpec {
|
||||
channels: 2,
|
||||
sample_rate: 48_000,
|
||||
bits_per_sample: 16,
|
||||
sample_format: SampleFormat::Int,
|
||||
};
|
||||
let mut writer = WavWriter::create(&path, spec).unwrap();
|
||||
// One 2ch loopback frame [0.4, 0.6] → R = downmix 0.5; mic 0.2 → L.
|
||||
let loop_bytes: Vec<u8> = [0.4f32, 0.6].iter().flat_map(|s| s.to_le_bytes()).collect();
|
||||
let n = write_wav_split(&mut writer, &loop_bytes, &format, &[0.2]).unwrap();
|
||||
writer.finalize().unwrap();
|
||||
assert_eq!(n, 1);
|
||||
|
||||
let mut reader = hound::WavReader::open(&path).unwrap();
|
||||
let s: Vec<i16> = reader.samples::<i16>().map(|x| x.unwrap()).collect();
|
||||
assert!((s[0] as f32 / i16::MAX as f32 - 0.2).abs() < 0.001); // L = mic
|
||||
assert!((s[1] as f32 / i16::MAX as f32 - 0.5).abs() < 0.001); // R = loopback downmix
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn f32_to_i16_clamps_and_scales() {
|
||||
assert_eq!(f32_to_i16(0.0), 0);
|
||||
|
||||
+348
-23
@@ -242,6 +242,66 @@ fn diarizer_from_installed_models() -> Option<SherpaDiarizer> {
|
||||
}
|
||||
}
|
||||
|
||||
// RETIRED (2026-07-14): the Phase 3 masked-mono attribution (`MicActivity`
|
||||
// timeline + `mic_activity.json` + `mask_ranges` + `phase3_attribute`).
|
||||
// Why it existed: to give per-stream "You" vs "Speaker" attribution when
|
||||
// `audio.wav` was a summed mono mix — record a live mic-speech timeline, then
|
||||
// diarize the mix with the mic's ranges masked out so clustering saw only the
|
||||
// far side. Why retired: it was fragile on reprocess (re-aligning a sidecar
|
||||
// timeline against a mono mix; collapsed to a single speaker), and the
|
||||
// dual-channel split layout below makes the mic/far-side separation intrinsic
|
||||
// to the file instead. Commits: 185489f / fd9311c / 0b5a85f (add), and this
|
||||
// change (remove). Revisit there if a summed-only capture path is ever needed.
|
||||
|
||||
/// Split attribution (FR-SPK, supersedes `phase3_attribute`): diarize the far
|
||||
/// side (right/loopback channel) into `Speaker N`, take "You" straight from
|
||||
/// left-channel (mic) voice activity, merge + assign + name. Shared by
|
||||
/// `stop_recording` and `reprocess_transcript`; both read it back from the file
|
||||
/// so they agree. `None` if the far-side pass fails (caller falls back).
|
||||
async fn attribute_split(
|
||||
diarizer: Arc<dyn Diarizer>,
|
||||
wav_path: PathBuf,
|
||||
segments: &mut [TranscriptSegment],
|
||||
) -> Option<HashMap<String, String>> {
|
||||
let diarizer_for_task = diarizer.clone();
|
||||
let result = tauri::async_runtime::spawn_blocking(move || {
|
||||
// Right channel = loopback/far side; diarize it alone (mic never in it).
|
||||
let far = crate::audio::read_wav_channel_16k(&wav_path, 1).map_err(|e| e.to_string())?;
|
||||
let far_spans = diarizer_for_task
|
||||
.diarize_samples(far)
|
||||
.map_err(|e| e.to_string())?;
|
||||
// Left channel = mic; its voice activity is "You".
|
||||
let mic = crate::audio::read_wav_channel_16k(&wav_path, 0).map_err(|e| e.to_string())?;
|
||||
Ok::<_, String>((far_spans, crate::audio::vad_spans(&mic)))
|
||||
})
|
||||
.await;
|
||||
let (far_spans, you_spans) = match result {
|
||||
Ok(Ok(v)) => v,
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!("split attribution failed: {e}");
|
||||
return None;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("split attribution task failed: {e}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let mut spans: Vec<SpeakerSpan> = you_spans
|
||||
.iter()
|
||||
.map(|&(start_ms, end_ms)| SpeakerSpan {
|
||||
start_ms,
|
||||
end_ms,
|
||||
speaker: "You".to_string(),
|
||||
})
|
||||
.collect();
|
||||
spans.extend(far_spans);
|
||||
spans.sort_by_key(|s| s.start_ms);
|
||||
diarizer.assign(segments, &spans);
|
||||
// Uniform naming: "You" -> "You", far speakers -> "Speaker 2", "Speaker 3"….
|
||||
let labels = crate::diarization::voiceprint::first_appearance_order(&spans);
|
||||
Some(crate::diarization::voiceprint::build_name_map(&labels, "You"))
|
||||
}
|
||||
|
||||
/// The distinct speakers seen in `segments` so far, in first-appearance order,
|
||||
/// with any display names applied (T4.3/T4.4, FR-SPK-2/5). Falls back to the
|
||||
/// single pre-diarization "S1" placeholder if no segments exist yet.
|
||||
@@ -341,11 +401,11 @@ pub async fn start_recording(
|
||||
// drops).
|
||||
let (capture, mic_capture, mic_voice_sample) = if settings.microphone_enabled {
|
||||
// The mixer sums both streams for the live transcript; the bridge carries
|
||||
// the mic into the loopback thread so the recorded WAV holds both sides
|
||||
// at native quality (FR-CAP-7).
|
||||
// the mic into the loopback thread. The loopback writes stereo split
|
||||
// (mic-left/loopback-right) so the two sides stay separate (FR-SPK).
|
||||
let bridge = crate::audio::MicBridge::shared();
|
||||
// A few seconds of raw mic audio for the post-stop voiceprint match
|
||||
// (bug: mic speaker mislabeled "S1"/"S2" instead of "You").
|
||||
// A few seconds of raw mic audio for the summed-recording voiceprint
|
||||
// fallback (imports / mic-off diarization).
|
||||
let voice_sample = crate::audio::VoiceSample::new(MIC_VOICEPRINT_SAMPLES);
|
||||
let (loop_sink, mic_sink) = crate::audio::spawn_mixer(frame_tx);
|
||||
let capture = WasapiCapture
|
||||
@@ -512,6 +572,7 @@ pub async fn start_recording(
|
||||
let wav_path_for_diar = wav_path.clone();
|
||||
let segments_for_diar = segments.clone();
|
||||
let names_for_diar = speaker_names.clone();
|
||||
let voice_sample_for_diar = mic_voice_sample.clone();
|
||||
tauri::async_runtime::spawn(async move {
|
||||
// ponytail: reprocesses the whole recording-so-far each tick
|
||||
// rather than incremental/windowed segmentation — sherpa-onnx's
|
||||
@@ -552,16 +613,59 @@ pub async fn start_recording(
|
||||
}
|
||||
};
|
||||
|
||||
let speakers = {
|
||||
let (speakers, changed) = {
|
||||
let mut segs = segments_for_diar.lock().unwrap_or_else(|e| e.into_inner());
|
||||
// Snapshot prior labels so only segments whose speaker
|
||||
// actually changed this pass get re-emitted (ids are stable;
|
||||
// the frontend replaces by id).
|
||||
let before: HashMap<u64, String> =
|
||||
segs.iter().map(|s| (s.id, s.speaker.clone())).collect();
|
||||
diarizer.assign(&mut segs, &spans);
|
||||
|
||||
// Live "You": match the mic voiceprint against this pass's
|
||||
// clusters. Clusters re-shuffle every tick so match every
|
||||
// tick; never overwrite a name already set (user rename or a
|
||||
// prior pass) — same guard as the post-stop pass.
|
||||
if let Some(voice_sample) = &voice_sample_for_diar {
|
||||
let mic_samples = voice_sample.samples();
|
||||
match crate::diarization::voiceprint::match_mic_speaker(
|
||||
&diarization_embedding_model_file(),
|
||||
&mic_samples,
|
||||
&wav_path_for_diar,
|
||||
&spans,
|
||||
) {
|
||||
Ok(auto_names) => {
|
||||
let mut names =
|
||||
names_for_diar.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for (label, name) in auto_names {
|
||||
names.entry(label).or_insert(name);
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!("live voiceprint match failed: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
let names = names_for_diar.lock().unwrap_or_else(|e| e.into_inner());
|
||||
speaker_infos_from_segments(&segs, &names)
|
||||
let changed: Vec<TranscriptSegment> = segs
|
||||
.iter()
|
||||
.filter(|s| before.get(&s.id) != Some(&s.speaker))
|
||||
.cloned()
|
||||
.collect();
|
||||
(speaker_infos_from_segments(&segs, &names), changed)
|
||||
};
|
||||
let _ = app_for_diar.emit(
|
||||
"diarization://updated",
|
||||
serde_json::json!({ "meetingId": meeting_id_for_diar, "speakers": speakers }),
|
||||
);
|
||||
// Re-emit relabeled committed segments so the live transcript
|
||||
// reflects the refined speakers without a new event type
|
||||
// (docs/04-api-contracts.md: transcript://segment is replace-by-id).
|
||||
for segment in &changed {
|
||||
let _ = app_for_diar.emit(
|
||||
"transcript://segment",
|
||||
serde_json::json!({ "meetingId": meeting_id_for_diar, "segment": segment }),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -581,6 +685,13 @@ pub async fn start_recording(
|
||||
diarizer,
|
||||
speaker_names,
|
||||
mic_voice_sample,
|
||||
// Mic on → the loopback writer records split (mic-left/loopback-right);
|
||||
// mic off → the summed loopback-only file (FR-SPK).
|
||||
audio_layout: if settings.microphone_enabled {
|
||||
"split"
|
||||
} else {
|
||||
"summed"
|
||||
},
|
||||
manual_notes: Arc::new(StdMutex::new(ManualNotes::default())),
|
||||
});
|
||||
drop(guard);
|
||||
@@ -638,11 +749,28 @@ pub async fn stop_recording(
|
||||
.unwrap_or_default();
|
||||
let segment_count = segments.len();
|
||||
|
||||
// T4.1/4.2: one authoritative diarization pass over the now-complete
|
||||
// recording (ADR-0005's "post-stop pass"), refining whatever the live
|
||||
// provisional passes (T4.3) produced. Skipped if diarization models
|
||||
// aren't installed — `speaker_infos_from_segments` then falls back to
|
||||
// the single pre-diarization "S1" placeholder, same as before Phase 4.
|
||||
// Speaker attribution over the now-complete recording.
|
||||
// FR-SPK: a split recording keeps the mic and far side on separate channels,
|
||||
// so diarize the far side (right channel) alone → "Speaker N" and take "You"
|
||||
// from left-channel voice activity. Falls back to the whole-signal pass
|
||||
// (ADR-0005) + voiceprint "You" for summed recordings (mic off) or if the
|
||||
// split pass fails.
|
||||
let mut speaker_names: HashMap<String, String> = HashMap::new();
|
||||
let mut attributed = false;
|
||||
if session.audio_layout == "split" {
|
||||
if let Some(diarizer) = session.diarizer.clone() {
|
||||
if let Some(names) =
|
||||
attribute_split(diarizer, session.wav_path.clone(), &mut segments).await
|
||||
{
|
||||
speaker_names = names;
|
||||
attributed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !attributed {
|
||||
// Fallback: one whole-signal pass, then voiceprint-match the mic against
|
||||
// the clusters to auto-label "You" (never overriding a user-set name).
|
||||
let mut final_spans: Option<Vec<SpeakerSpan>> = None;
|
||||
if let Some(diarizer) = session.diarizer.clone() {
|
||||
let diarizer_for_task = diarizer.clone();
|
||||
@@ -658,11 +786,6 @@ pub async fn stop_recording(
|
||||
Err(e) => tracing::warn!("final diarization task failed: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
// Bug fix: identify which diarized cluster is the mic (voiceprint match
|
||||
// against the mic-only sample) and auto-label it "You" — otherwise the
|
||||
// mic speaker is just whichever cluster sherpa-onnx happened to call
|
||||
// "S1". Never overrides a name the user already set live (T4.4).
|
||||
if let (Some(voice_sample), Some(spans)) = (&session.mic_voice_sample, &final_spans) {
|
||||
let mic_samples = voice_sample.samples();
|
||||
match crate::diarization::voiceprint::match_mic_speaker(
|
||||
@@ -693,12 +816,13 @@ pub async fn stop_recording(
|
||||
Err(e) => tracing::warn!("mic voiceprint match failed: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
let speaker_names = session
|
||||
speaker_names = session
|
||||
.speaker_names
|
||||
.lock()
|
||||
.map(|g| g.clone())
|
||||
.unwrap_or_default();
|
||||
}
|
||||
|
||||
let speakers = speaker_infos_from_segments(&segments, &speaker_names);
|
||||
let backend_used = session
|
||||
.active_backend
|
||||
@@ -725,6 +849,7 @@ pub async fn stop_recording(
|
||||
language,
|
||||
backend_used: Some(backend_used),
|
||||
model_used: Some(session.model_id.clone()),
|
||||
audio_layout: Some(session.audio_layout.to_string()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -757,8 +882,12 @@ pub async fn stop_recording(
|
||||
|
||||
// ADR-0009: delete the working WAV only after the transcript is finalized
|
||||
// above, and only when retention is off.
|
||||
let voiceprint_path = meeting_dir(&meeting_id).join("voiceprint.wav");
|
||||
if !session.retention {
|
||||
let _ = std::fs::remove_file(&session.wav_path);
|
||||
// voiceprint.wav lives and dies with audio.wav (ADR-0009); never written
|
||||
// when retention is off, but remove defensively regardless.
|
||||
let _ = std::fs::remove_file(&voiceprint_path);
|
||||
} else if crate::vault::is_unlocked() {
|
||||
// Seal the retained recording at rest when the vault is unlocked (T8.8).
|
||||
// Runs before the sync enqueue below, so any uploaded copy is ciphertext.
|
||||
@@ -769,6 +898,25 @@ pub async fn stop_recording(
|
||||
}
|
||||
}
|
||||
|
||||
// FR-SPK: persist the mic voiceprint alongside a retained recording so
|
||||
// reprocess_transcript can re-identify "You" after it re-clusters. Retained
|
||||
// audio of the user's own voice, so it's gated on the same ADR-0009
|
||||
// consent/retention as audio.wav and sealed at rest the same way.
|
||||
if session.retention {
|
||||
if let Some(voice_sample) = &session.mic_voice_sample {
|
||||
let samples = voice_sample.samples();
|
||||
if let Err(e) = crate::audio::write_wav_mono_16k(&voiceprint_path, &samples) {
|
||||
tracing::warn!("failed to persist voiceprint.wav: {e}");
|
||||
} else if crate::vault::is_unlocked() {
|
||||
if let Ok(raw) = std::fs::read(&voiceprint_path) {
|
||||
if let Ok(sealed) = crate::vault::seal(&raw) {
|
||||
let _ = std::fs::write(&voiceprint_path, sealed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sync-on-finalize (T9.5, FR-SYNC-5): enqueue configured artifacts for
|
||||
// finalize-trigger targets and pump in the background so stop returns
|
||||
// promptly. Runs after the WAV-retention decision above, so a deleted
|
||||
@@ -1642,6 +1790,12 @@ pub async fn remove_model(id: String) -> WaResult<()> {
|
||||
|
||||
/// Batch re-transcribe a finished meeting with a different (typically larger)
|
||||
/// model (T3.8, FR-TRX-3). Only works if the meeting's audio was retained.
|
||||
///
|
||||
/// Re-diarizes from scratch: the fresh transcription is re-clustered and, if a
|
||||
/// `voiceprint.wav` was retained, the mic cluster is re-labeled "You" (FR-SPK).
|
||||
/// The meeting's stored speaker names are **discarded** — they key to the
|
||||
/// original run's labels, which no longer exist after re-clustering, so any
|
||||
/// names the user typed on the first pass are intentionally lost here.
|
||||
#[tauri::command]
|
||||
pub async fn reprocess_transcript(
|
||||
app: AppHandle,
|
||||
@@ -1679,7 +1833,7 @@ pub async fn reprocess_transcript(
|
||||
let requested_language =
|
||||
normalize_language(language.as_deref().or(meeting.language.as_deref()))
|
||||
.map(|s| s.to_string());
|
||||
let (segments, resolved_language) = tauri::async_runtime::spawn_blocking({
|
||||
let (mut segments, resolved_language) = tauri::async_runtime::spawn_blocking({
|
||||
let wav_path = wav_path.clone();
|
||||
move || {
|
||||
let (transcriber, _used) =
|
||||
@@ -1695,6 +1849,75 @@ pub async fn reprocess_transcript(
|
||||
.map_err(|e| WaError::new("transcription", e.to_string()))?
|
||||
.map_err(|e| WaError::new("transcription", e.to_string()))?;
|
||||
|
||||
// FR-SPK: re-diarize the fresh transcript. The meeting's stored labels key
|
||||
// to the original clustering and are meaningless now — rebuild from the
|
||||
// audio. Missing models degrade to the single "S1" placeholder, same as
|
||||
// live/import.
|
||||
let diarizer: Option<Arc<dyn Diarizer>> =
|
||||
tauri::async_runtime::spawn_blocking(diarizer_from_installed_models)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|d| Arc::new(d) as Arc<dyn Diarizer>);
|
||||
// Rebuild the name map fresh (stale labels are discarded either way).
|
||||
// Prefer the Phase 3 mic timeline when it was persisted (mic = "You",
|
||||
// diarize the far side masked); otherwise fall back to the whole-signal pass
|
||||
// + voiceprint match (imports, pre-Phase-3 recordings). No models → raw "S1".
|
||||
let mut speaker_names = HashMap::new();
|
||||
let is_split = meeting.audio_layout.as_deref() == Some("split");
|
||||
match diarizer {
|
||||
Some(diarizer) if is_split => {
|
||||
// Split recording: diarize the far side (R channel), "You" from the
|
||||
// mic (L) channel — recomputed from the file, matching stop's result.
|
||||
if let Some(names) = attribute_split(diarizer, wav_path.clone(), &mut segments).await {
|
||||
speaker_names = names;
|
||||
}
|
||||
}
|
||||
Some(diarizer) => {
|
||||
// Summed recording: whole-signal pass + voiceprint "You" match.
|
||||
let mut spans: Option<Vec<SpeakerSpan>> = None;
|
||||
let diarizer_for_task = diarizer.clone();
|
||||
let wp = wav_path.clone();
|
||||
match tauri::async_runtime::spawn_blocking(move || diarizer_for_task.diarize(&wp)).await
|
||||
{
|
||||
Ok(Ok(s)) => {
|
||||
diarizer.assign(&mut segments, &s);
|
||||
spans = Some(s);
|
||||
}
|
||||
Ok(Err(e)) => tracing::warn!("reprocess diarization pass failed: {e}"),
|
||||
Err(e) => tracing::warn!("reprocess diarization task failed: {e}"),
|
||||
}
|
||||
let voiceprint_path = meeting_dir(&meeting_id).join("voiceprint.wav");
|
||||
if let (Some(spans), true) = (&spans, voiceprint_path.exists()) {
|
||||
match crate::audio::read_wav_mono_16k(&voiceprint_path) {
|
||||
Ok(mic_samples) => match crate::diarization::voiceprint::match_mic_speaker(
|
||||
&diarization_embedding_model_file(),
|
||||
&mic_samples,
|
||||
&wav_path,
|
||||
spans,
|
||||
) {
|
||||
Ok(names) => speaker_names = names,
|
||||
Err(e) => tracing::warn!("reprocess voiceprint match failed: {e}"),
|
||||
},
|
||||
Err(e) => tracing::warn!("failed to read voiceprint.wav: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
None => {} // no diarization models: raw "S1" labels
|
||||
}
|
||||
let speakers = speaker_infos_from_segments(&segments, &speaker_names);
|
||||
|
||||
// Drop the previous run's speaker rows before re-inserting the fresh set:
|
||||
// the old labels key to the discarded clustering, so leaving them stranded
|
||||
// in the DB makes the Participants pane show ghosts (e.g. 83 old labels
|
||||
// when the new transcript has 5). finalize_meeting below re-upserts
|
||||
// `speakers`. (FR-SPK)
|
||||
state
|
||||
.store
|
||||
.clear_speakers(&meeting_id)
|
||||
.await
|
||||
.map_err(|e| WaError::new("storage", e.to_string()))?;
|
||||
|
||||
let duration_secs = segments
|
||||
.last()
|
||||
.map(|s| (s.end_ms / 1000) as i64)
|
||||
@@ -1710,12 +1933,13 @@ pub async fn reprocess_transcript(
|
||||
&meeting_id,
|
||||
FinalizeMeeting {
|
||||
segments: segments.clone(),
|
||||
speakers: meeting.speakers.clone(),
|
||||
speakers: speakers.clone(),
|
||||
duration_secs,
|
||||
recorded: meeting.recorded,
|
||||
language: resolved_language,
|
||||
backend_used: Some(backend.as_str().to_string()),
|
||||
model_used: Some(model),
|
||||
audio_layout: None, // reprocess preserves the recorded layout
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1726,7 +1950,7 @@ pub async fn reprocess_transcript(
|
||||
let manual_notes = load_manual_notes(&meeting_id);
|
||||
let notes_md = crate::notes::MarkdownNotes.merge(
|
||||
&segments,
|
||||
&meeting.speakers,
|
||||
&speakers,
|
||||
&manual_notes,
|
||||
None,
|
||||
template.as_ref(),
|
||||
@@ -1877,6 +2101,7 @@ pub async fn import_media(
|
||||
language: resolved_language,
|
||||
backend_used: Some(backend.as_str().to_string()),
|
||||
model_used: Some(model_id.clone()),
|
||||
audio_layout: Some("summed".to_string()), // single-source import
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -1991,6 +2216,7 @@ pub async fn resume_transcription(
|
||||
language: resolved_language,
|
||||
backend_used: Some(BackendId::Cpu.as_str().to_string()),
|
||||
model_used: Some(model_id),
|
||||
audio_layout: None, // recovery preserves the recorded layout
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2161,6 +2387,54 @@ pub async fn recording_playback_path(meeting_id: MeetingId) -> WaResult<String>
|
||||
/// `audio.wav`, decrypts it in memory if sealed (T8.8), and streams the PCM to
|
||||
/// the `<audio>` element with byte-range support for seeking. The plaintext
|
||||
/// never touches the filesystem, so playback can't undermine encryption at rest.
|
||||
/// Fold a WAV's channels to their per-frame average, written back across
|
||||
/// `out_channels` identical output channels (FR-SPK). Playback uses
|
||||
/// `out_channels = 1` so a split (mic-left/loopback-right) recording plays both
|
||||
/// sources in both ears; bundle export uses `2` (dual-mono) so the shared file
|
||||
/// looks like a normal stereo file. Returns `None` when there's nothing to do
|
||||
/// (already mono → mono) or the WAV is unparseable (caller keeps the original).
|
||||
/// ponytail: re-decodes the whole file each call; cache if it ever gets heavy.
|
||||
fn fold_wav(wav: &[u8], out_channels: u16) -> Option<Vec<u8>> {
|
||||
let mut reader = hound::WavReader::new(std::io::Cursor::new(wav)).ok()?;
|
||||
let spec = reader.spec();
|
||||
let ch = spec.channels as usize;
|
||||
if ch as u16 == out_channels && ch <= 1 {
|
||||
return None; // already mono and mono requested — nothing to do
|
||||
}
|
||||
let out_spec = hound::WavSpec {
|
||||
channels: out_channels,
|
||||
..spec
|
||||
};
|
||||
let mut out = std::io::Cursor::new(Vec::new());
|
||||
{
|
||||
let mut writer = hound::WavWriter::new(&mut out, out_spec).ok()?;
|
||||
match (spec.sample_format, spec.bits_per_sample) {
|
||||
(hound::SampleFormat::Int, 16) => {
|
||||
let s: Vec<i16> = reader.samples::<i16>().map_while(Result::ok).collect();
|
||||
for frame in s.chunks(ch) {
|
||||
let avg = (frame.iter().map(|&x| x as i32).sum::<i32>()
|
||||
/ frame.len() as i32) as i16;
|
||||
for _ in 0..out_channels {
|
||||
writer.write_sample(avg).ok()?;
|
||||
}
|
||||
}
|
||||
}
|
||||
(hound::SampleFormat::Float, 32) => {
|
||||
let s: Vec<f32> = reader.samples::<f32>().map_while(Result::ok).collect();
|
||||
for frame in s.chunks(ch) {
|
||||
let avg = frame.iter().sum::<f32>() / frame.len() as f32;
|
||||
for _ in 0..out_channels {
|
||||
writer.write_sample(avg).ok()?;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => return None,
|
||||
}
|
||||
writer.finalize().ok()?;
|
||||
}
|
||||
Some(out.into_inner())
|
||||
}
|
||||
|
||||
pub(crate) fn serve_recording(
|
||||
request: &tauri::http::Request<Vec<u8>>,
|
||||
) -> tauri::http::Response<Vec<u8>> {
|
||||
@@ -2185,6 +2459,9 @@ pub(crate) fn serve_recording(
|
||||
Ok(p) => p,
|
||||
Err(_) => return fail(StatusCode::FORBIDDEN), // sealed + vault locked
|
||||
};
|
||||
// Fold stereo → mono so a split (mic-left/loopback-right) recording plays
|
||||
// both sources in both ears (FR-SPK); mono/unparseable passes through.
|
||||
let plain = fold_wav(&plain, 1).unwrap_or(plain);
|
||||
let total = plain.len();
|
||||
|
||||
let base = || {
|
||||
@@ -2410,6 +2687,7 @@ async fn import_one_bundle(state: &State<'_, AppState>, dir: &Path) -> WaResult<
|
||||
language: bundle.language.clone(),
|
||||
backend_used: bundle.backend_used.clone(),
|
||||
model_used: bundle.model_used.clone(),
|
||||
audio_layout: bundle.audio_layout.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2538,10 +2816,17 @@ async fn export_meeting_to(
|
||||
std::fs::create_dir_all(dest_path)
|
||||
.map_err(|e| WaError::new("export", e.to_string()))?;
|
||||
let source_dir = meeting_dir(meeting_id);
|
||||
// Audio is stored plaintext — copy as-is.
|
||||
// Audio: decrypt (T8.8) and fold to dual-mono so the shared file is a
|
||||
// normal stereo file, not a mic-left/loopback-right split (FR-SPK).
|
||||
// The manifest records this exported copy as `audio_layout: summed`.
|
||||
let audio_src = source_dir.join("audio.wav");
|
||||
if audio_src.exists() {
|
||||
std::fs::copy(&audio_src, dest_path.join("audio.wav"))
|
||||
let raw =
|
||||
std::fs::read(&audio_src).map_err(|e| WaError::new("export", e.to_string()))?;
|
||||
let plain =
|
||||
crate::vault::open(&raw).map_err(|e| WaError::new("export", e.to_string()))?;
|
||||
let out = fold_wav(&plain, 2).unwrap_or_else(|| plain.to_vec());
|
||||
std::fs::write(dest_path.join("audio.wav"), out)
|
||||
.map_err(|e| WaError::new("export", e.to_string()))?;
|
||||
}
|
||||
// transcript.json may be vault-sealed; export the decrypted content
|
||||
@@ -2585,6 +2870,9 @@ async fn export_meeting_to(
|
||||
tags: meeting.tags.clone(),
|
||||
speakers: meeting.speakers.clone(),
|
||||
action_items: meeting.action_items.clone(),
|
||||
// Export downmixes a split recording to dual-mono below, so the
|
||||
// shared copy is a normal "summed" file (FR-STORE-4).
|
||||
audio_layout: Some("summed".to_string()),
|
||||
};
|
||||
let manifest = serde_json::to_string_pretty(&bundle)
|
||||
.map_err(|e| WaError::new("export", e.to_string()))?;
|
||||
@@ -4675,6 +4963,41 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fold_wav_downmixes_and_dual_monos() {
|
||||
// 2ch 16-bit WAV, one frame [1000, 3000] → average 2000.
|
||||
let spec = hound::WavSpec {
|
||||
channels: 2,
|
||||
sample_rate: 16_000,
|
||||
bits_per_sample: 16,
|
||||
sample_format: hound::SampleFormat::Int,
|
||||
};
|
||||
let mut buf = std::io::Cursor::new(Vec::new());
|
||||
{
|
||||
let mut w = hound::WavWriter::new(&mut buf, spec).unwrap();
|
||||
w.write_sample(1000i16).unwrap();
|
||||
w.write_sample(3000i16).unwrap();
|
||||
w.finalize().unwrap();
|
||||
}
|
||||
let wav = buf.into_inner();
|
||||
|
||||
let mono = fold_wav(&wav, 1).unwrap();
|
||||
let mut r = hound::WavReader::new(std::io::Cursor::new(mono)).unwrap();
|
||||
assert_eq!(r.spec().channels, 1);
|
||||
assert_eq!(
|
||||
r.samples::<i16>().map(|x| x.unwrap()).collect::<Vec<_>>(),
|
||||
vec![2000]
|
||||
);
|
||||
|
||||
let dual = fold_wav(&wav, 2).unwrap();
|
||||
let mut r2 = hound::WavReader::new(std::io::Cursor::new(dual)).unwrap();
|
||||
assert_eq!(r2.spec().channels, 2);
|
||||
assert_eq!(
|
||||
r2.samples::<i16>().map(|x| x.unwrap()).collect::<Vec<_>>(),
|
||||
vec![2000, 2000] // dual-mono: both channels the mixed sample
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speaker_infos_lists_distinct_speakers_in_first_appearance_order() {
|
||||
let segments = vec![segment("S2"), segment("S1"), segment("S2")];
|
||||
@@ -4771,6 +5094,7 @@ mod tests {
|
||||
language: None,
|
||||
backend_used: None,
|
||||
model_used: None,
|
||||
audio_layout: None,
|
||||
segments: Vec::new(),
|
||||
speakers: vec![
|
||||
SpeakerInfo {
|
||||
@@ -5055,6 +5379,7 @@ mod tests {
|
||||
language: None,
|
||||
backend_used: None,
|
||||
model_used: None,
|
||||
audio_layout: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -23,6 +23,11 @@ pub enum DiarError {
|
||||
pub trait Diarizer: Send + Sync {
|
||||
/// Partition audio into speaker spans.
|
||||
fn diarize(&self, wav: &Path) -> Result<Vec<SpeakerSpan>, DiarError>;
|
||||
/// Partition already-loaded 16kHz mono samples into speaker spans. Phase 3
|
||||
/// (FR-SPK) diarizes a *masked* far-side signal (the mic's "You" ranges
|
||||
/// zeroed), so the caller loads and masks the samples itself rather than
|
||||
/// pointing at the raw WAV.
|
||||
fn diarize_samples(&self, samples: Vec<f32>) -> Result<Vec<SpeakerSpan>, DiarError>;
|
||||
/// Assign speaker labels to transcript segments by overlap with spans.
|
||||
fn assign(&self, segments: &mut [TranscriptSegment], spans: &[SpeakerSpan]);
|
||||
}
|
||||
@@ -70,7 +75,13 @@ impl SherpaDiarizer {
|
||||
// sherpa-onnx to pick the cluster count itself from `threshold`
|
||||
// instead of forcing a fixed number of speakers.
|
||||
num_clusters: Some(-1),
|
||||
threshold: Some(0.5),
|
||||
// Tuning evidence: 0.5 over the summed mic+loopback mono WAV
|
||||
// exploded a 2-speaker conversation into 83 clusters (overlapped
|
||||
// speech + short pyannote chunks yield unstable embeddings). sherpa
|
||||
// fast-clustering: larger threshold → fewer clusters. 0.7 lands a
|
||||
// 2-person meeting near 2–4. ponytail: code constant, no settings
|
||||
// knob until 0.7 proves insufficient in the field.
|
||||
threshold: Some(0.7),
|
||||
..Default::default()
|
||||
};
|
||||
let engine = sherpa_rs::diarize::Diarize::new(segmentation_model, embedding_model, config)
|
||||
@@ -86,11 +97,28 @@ impl Diarizer for SherpaDiarizer {
|
||||
fn diarize(&self, wav: &Path) -> Result<Vec<SpeakerSpan>, DiarError> {
|
||||
let samples =
|
||||
crate::audio::read_wav_mono_16k(wav).map_err(|e| DiarError::Run(e.to_string()))?;
|
||||
self.diarize_samples(samples)
|
||||
}
|
||||
|
||||
fn diarize_samples(&self, samples: Vec<f32>) -> Result<Vec<SpeakerSpan>, DiarError> {
|
||||
// Guard the native FFI against degenerate input (a fully-masked far side
|
||||
// is all zeros; a tiny buffer has no usable window). sherpa can crash
|
||||
// rather than error on these, so skip cleanly instead.
|
||||
if samples.len() < MIN_DIARIZE_SAMPLES {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut engine = self.engine.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let segments = engine
|
||||
.compute(samples, None)
|
||||
.map_err(|e| DiarError::Run(e.to_string()))?;
|
||||
Ok(segments.into_iter().map(segment_to_span).collect())
|
||||
// Drop sub-700ms spans: their embeddings are unstable and only cause
|
||||
// label churn in assign_by_overlap (which keeps the prior label when no
|
||||
// span overlaps). See MIN_SPAN_MS.
|
||||
Ok(segments
|
||||
.into_iter()
|
||||
.map(segment_to_span)
|
||||
.filter(span_long_enough)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn assign(&self, segments: &mut [TranscriptSegment], spans: &[SpeakerSpan]) {
|
||||
@@ -98,6 +126,23 @@ impl Diarizer for SherpaDiarizer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimum span duration kept from a diarization pass. Sub-second pyannote
|
||||
/// chunks carry unstable ERes2Net embeddings and only churn labels.
|
||||
#[cfg(feature = "diarization")]
|
||||
const MIN_SPAN_MS: u64 = 700;
|
||||
|
||||
/// Below this many 16kHz samples (~1s) the diarizer skips the sherpa FFI: a
|
||||
/// fully-masked far side is all zeros and a tiny buffer has no usable
|
||||
/// segmentation window, either of which can crash the native code.
|
||||
#[cfg(feature = "diarization")]
|
||||
const MIN_DIARIZE_SAMPLES: usize = 16_000;
|
||||
|
||||
/// A span is kept only if it lasts at least [`MIN_SPAN_MS`].
|
||||
#[cfg(feature = "diarization")]
|
||||
fn span_long_enough(span: &SpeakerSpan) -> bool {
|
||||
span.end_ms.saturating_sub(span.start_ms) >= MIN_SPAN_MS
|
||||
}
|
||||
|
||||
/// sherpa-onnx speaker indices are 0-based; WA's internal labels are 1-based ("S1"…).
|
||||
#[cfg(feature = "diarization")]
|
||||
fn segment_to_span(seg: sherpa_rs::diarize::Segment) -> SpeakerSpan {
|
||||
@@ -184,6 +229,22 @@ mod tests {
|
||||
assert_eq!(span.speaker, "S1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn span_long_enough_filters_sub_minimum_spans() {
|
||||
let short = SpeakerSpan {
|
||||
start_ms: 0,
|
||||
end_ms: MIN_SPAN_MS - 1,
|
||||
speaker: "S1".to_string(),
|
||||
};
|
||||
let exact = SpeakerSpan {
|
||||
start_ms: 0,
|
||||
end_ms: MIN_SPAN_MS,
|
||||
speaker: "S1".to_string(),
|
||||
};
|
||||
assert!(!span_long_enough(&short));
|
||||
assert!(span_long_enough(&exact));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_surfaces_a_load_error_for_missing_models_instead_of_panicking() {
|
||||
let result =
|
||||
|
||||
@@ -110,8 +110,9 @@ pub fn match_mic_speaker(
|
||||
}
|
||||
|
||||
/// Distinct speaker labels in first-appearance order — spans come back from
|
||||
/// the diarizer already sorted by start time.
|
||||
fn first_appearance_order(spans: &[SpeakerSpan]) -> Vec<String> {
|
||||
/// the diarizer already sorted by start time. `pub(crate)` so the Phase 3
|
||||
/// per-stream path can reuse it to name its merged You + far-side spans.
|
||||
pub(crate) fn first_appearance_order(spans: &[SpeakerSpan]) -> Vec<String> {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
spans
|
||||
.iter()
|
||||
@@ -152,8 +153,12 @@ fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
|
||||
|
||||
/// `mic_label` -> "You"; every other label, in first-appearance order ->
|
||||
/// "Speaker 2", "Speaker 3", … (numbering starts at 2 — "You" stands in for
|
||||
/// "Speaker 1" without ever being called that).
|
||||
fn build_name_map(labels_in_order: &[String], mic_label: &str) -> HashMap<String, String> {
|
||||
/// "Speaker 1" without ever being called that). `pub(crate)` so Phase 3 reuses
|
||||
/// it for its merged You + far-side spans, keeping naming uniform app-wide.
|
||||
pub(crate) fn build_name_map(
|
||||
labels_in_order: &[String],
|
||||
mic_label: &str,
|
||||
) -> HashMap<String, String> {
|
||||
let mut names = HashMap::new();
|
||||
let mut next_speaker_number = 2;
|
||||
for label in labels_in_order {
|
||||
|
||||
@@ -85,6 +85,10 @@ pub struct RecordingSession {
|
||||
/// auto-labeled "You" instead of a clustered "S1"/"S2". `None` when the
|
||||
/// mic is disabled (same conditions as `mic_capture`).
|
||||
pub mic_voice_sample: Option<Arc<audio::VoiceSample>>,
|
||||
/// `audio.wav` channel layout for this recording (FR-SPK): `"split"` (mic
|
||||
/// enabled → stereo mic-left/loopback-right) or `"summed"` (mic off). Passed
|
||||
/// to `finalize_meeting` so playback/diarization/export can branch on it.
|
||||
pub audio_layout: &'static str,
|
||||
/// Live notes redesign: raw user-authored notes accumulated *during* the
|
||||
/// recording (freeform text + per-moment annotations) — see
|
||||
/// `models::ManualNotes`. Mutated by `update_live_notes`/`set_segment_note`
|
||||
|
||||
@@ -214,6 +214,11 @@ pub struct MeetingBundle {
|
||||
pub tags: Vec<String>,
|
||||
pub speakers: Vec<SpeakerInfo>,
|
||||
pub action_items: Vec<ActionItem>,
|
||||
/// `audio.wav` channel layout (FR-SPK/FR-CAP): `"split"` or `"summed"`.
|
||||
/// `default` so bundles exported before this field deserialize as `None`
|
||||
/// (treated as `"summed"`).
|
||||
#[serde(default)]
|
||||
pub audio_layout: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
@@ -74,6 +74,10 @@ pub struct FinalizeMeeting {
|
||||
pub language: Option<String>,
|
||||
pub backend_used: Option<String>,
|
||||
pub model_used: Option<String>,
|
||||
/// How `audio.wav`'s channels are laid out (FR-SPK/FR-CAP): `"split"` =
|
||||
/// stereo mic-left/loopback-right, `"summed"`/`None` = mic mixed into every
|
||||
/// channel. `None` leaves the stored value unchanged (e.g. on reprocess).
|
||||
pub audio_layout: Option<String>,
|
||||
}
|
||||
|
||||
/// Full meeting detail: DB row + transcript + speakers + notes (`get_meeting`'s
|
||||
@@ -91,6 +95,10 @@ pub struct Meeting {
|
||||
pub language: Option<String>,
|
||||
pub backend_used: Option<String>,
|
||||
pub model_used: Option<String>,
|
||||
/// `audio.wav` channel layout (FR-SPK/FR-CAP): `"split"` (mic-left /
|
||||
/// loopback-right) or `"summed"`/`None` (mic mixed into every channel).
|
||||
/// Diarization, playback downmix, and bundle export branch on it.
|
||||
pub audio_layout: Option<String>,
|
||||
pub segments: Vec<TranscriptSegment>,
|
||||
pub speakers: Vec<SpeakerInfo>,
|
||||
pub notes_markdown: String,
|
||||
@@ -284,6 +292,14 @@ pub trait Store: Send + Sync {
|
||||
label: &str,
|
||||
name: &str,
|
||||
) -> Result<(), StoreError>;
|
||||
/// Delete every `speakers` row for a meeting (labels, names, participant
|
||||
/// links, merges). Used before a full re-diarization rebuild
|
||||
/// (`reprocess_transcript`, FR-SPK): the old labels key to the previous
|
||||
/// clustering and are meaningless once the audio is re-clustered, so the
|
||||
/// caller re-inserts the fresh set via `finalize_meeting`. Without this,
|
||||
/// stale labels from an over-split run linger in the DB (and the
|
||||
/// Participants pane) even though the transcript no longer references them.
|
||||
async fn clear_speakers(&self, id: &MeetingId) -> Result<(), StoreError>;
|
||||
/// Fold over-split speaker labels into one canonical label (T4.5,
|
||||
/// FR-SPK-3). Segment speaker IDs in storage are never rewritten
|
||||
/// (FR-SPK-5) — `get_meeting` resolves `from` labels to `into` when it
|
||||
@@ -734,7 +750,8 @@ impl Store for SqliteStore {
|
||||
let now = now_unix();
|
||||
sqlx::query(
|
||||
"UPDATE meetings SET status = 'ready', ended_at = ?, duration_secs = ?, recorded = ?,
|
||||
language = ?, backend_used = ?, model_used = ?, updated_at = ? WHERE id = ?",
|
||||
language = ?, backend_used = ?, model_used = ?,
|
||||
audio_layout = COALESCE(?, audio_layout), updated_at = ? WHERE id = ?",
|
||||
)
|
||||
.bind(now)
|
||||
.bind(s.duration_secs)
|
||||
@@ -742,6 +759,7 @@ impl Store for SqliteStore {
|
||||
.bind(&s.language)
|
||||
.bind(&s.backend_used)
|
||||
.bind(&s.model_used)
|
||||
.bind(&s.audio_layout)
|
||||
.bind(now)
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
@@ -857,7 +875,7 @@ impl Store for SqliteStore {
|
||||
|
||||
async fn get_meeting(&self, id: &MeetingId) -> Result<Meeting, StoreError> {
|
||||
let row = sqlx::query(
|
||||
"SELECT id, title, started_at, ended_at, duration_secs, status, recorded, language, backend_used, model_used, calendar_event_id, template_id
|
||||
"SELECT id, title, started_at, ended_at, duration_secs, status, recorded, language, backend_used, model_used, calendar_event_id, template_id, audio_layout
|
||||
FROM meetings WHERE id = ?",
|
||||
)
|
||||
.bind(id)
|
||||
@@ -928,6 +946,7 @@ impl Store for SqliteStore {
|
||||
language: row.get("language"),
|
||||
backend_used: row.get("backend_used"),
|
||||
model_used: row.get("model_used"),
|
||||
audio_layout: row.get("audio_layout"),
|
||||
segments,
|
||||
speakers,
|
||||
notes_markdown,
|
||||
@@ -1252,6 +1271,14 @@ impl Store for SqliteStore {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn clear_speakers(&self, id: &MeetingId) -> Result<(), StoreError> {
|
||||
sqlx::query("DELETE FROM speakers WHERE meeting_id = ?")
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn merge_speakers(
|
||||
&self,
|
||||
id: &MeetingId,
|
||||
@@ -2015,6 +2042,7 @@ mod tests {
|
||||
language: Some("fr".to_string()), // what auto-detect resolved to
|
||||
backend_used: Some("cpu".to_string()),
|
||||
model_used: Some("small-q5_1".to_string()),
|
||||
audio_layout: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2058,6 +2086,7 @@ mod tests {
|
||||
language: Some("en".to_string()),
|
||||
backend_used: Some("cpu".to_string()),
|
||||
model_used: Some("small-q5_1".to_string()),
|
||||
audio_layout: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "WhispAssist",
|
||||
"version": "0.5.2",
|
||||
"version": "0.6.0",
|
||||
"identifier": "bet.dou.whispassist",
|
||||
"build": {
|
||||
"frontendDist": "../dist",
|
||||
|
||||
@@ -594,6 +594,11 @@ export const events = {
|
||||
onFinalized: (
|
||||
cb: (p: { meetingId: string; segmentCount: number }) => void,
|
||||
): Promise<UnlistenFn> => listen("transcript://finalized", (e) => cb(e.payload as never)),
|
||||
// Live diarization refined the speaker list mid-recording (FR-SPK): updated
|
||||
// labels/display names, including the mic speaker resolved to "You".
|
||||
onDiarizationUpdated: (
|
||||
cb: (p: { meetingId: string; speakers: SpeakerInfo[] }) => void,
|
||||
): Promise<UnlistenFn> => listen("diarization://updated", (e) => cb(e.payload as never)),
|
||||
onLlmToken: (cb: (p: { meetingId: string; text: string }) => void): Promise<UnlistenFn> =>
|
||||
listen("llm://token", (e) => cb(e.payload as never)),
|
||||
onLlmDone: (cb: (p: { meetingId: string; summary: SummaryFile }) => void): Promise<UnlistenFn> =>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Recording state store (Svelte 5 runes-friendly via a small class).
|
||||
// Subscribes to recording/transcript events and exposes reactive state.
|
||||
|
||||
import { api, events, type TranscriptSegment, type MeetingId } from "../api";
|
||||
import { api, events, type TranscriptSegment, type SpeakerInfo, type MeetingId } from "../api";
|
||||
import { settings } from "./settings.svelte";
|
||||
import { SvelteMap } from "svelte/reactivity";
|
||||
|
||||
@@ -10,6 +10,9 @@ class RecordingStore {
|
||||
state = $state<"idle" | "recording" | "paused">("idle");
|
||||
elapsedMs = $state(0);
|
||||
segments = $state<TranscriptSegment[]>([]);
|
||||
/** Live speaker list from provisional diarization (FR-SPK): resolves labels
|
||||
* to display names ("You" for the mic speaker) for the live transcript. */
|
||||
speakers = $state<SpeakerInfo[]>([]);
|
||||
/** Whether audio is being retained as .wav for the in-flight meeting (ADR-0009). */
|
||||
retention = $state(false);
|
||||
/** Live system/loopback level for the waveform/meter (FR-CAP-5); 0 when not recording. */
|
||||
@@ -47,11 +50,16 @@ class RecordingStore {
|
||||
this.retention = p.record;
|
||||
});
|
||||
await events.onSegment(({ segment }) => {
|
||||
// Replace an interim segment with the same id, else append.
|
||||
// Replace an interim segment with the same id, else append. Live
|
||||
// diarization also re-emits committed segments here with a refined
|
||||
// speaker (FR-SPK) — same replace-by-id path applies.
|
||||
const i = this.segments.findIndex((s) => s.id === segment.id);
|
||||
if (i >= 0) this.segments[i] = segment;
|
||||
else this.segments.push(segment);
|
||||
});
|
||||
await events.onDiarizationUpdated(({ speakers }) => {
|
||||
this.speakers = speakers;
|
||||
});
|
||||
await events.onLevel(({ rms, peak, mic }) => {
|
||||
if (mic) {
|
||||
this.levelRmsMic = rms;
|
||||
@@ -68,6 +76,7 @@ class RecordingStore {
|
||||
|
||||
async start(title?: string, record = false, templateId?: string, calendarEventId?: string) {
|
||||
this.segments = [];
|
||||
this.speakers = [];
|
||||
this.retention = record;
|
||||
this.deviceNotice = null;
|
||||
this.notesText = "";
|
||||
@@ -101,6 +110,7 @@ class RecordingStore {
|
||||
if (this.meetingId) await api.cancelRecording(this.meetingId);
|
||||
this.meetingId = null;
|
||||
this.segments = [];
|
||||
this.speakers = [];
|
||||
this.state = "idle";
|
||||
this.levelRms = 0;
|
||||
this.levelPeak = 0;
|
||||
|
||||
@@ -524,7 +524,7 @@
|
||||
onclick={() => (selectedSegmentMs = open ? null : s.start_ms)}
|
||||
>
|
||||
<span class="ts">{fmtTs(s.start_ms)}</span>
|
||||
<strong>{speakerName(s.speaker)}:</strong>
|
||||
<strong>{speakerName(s.speaker, recording.speakers)}:</strong>
|
||||
{s.text}
|
||||
{#if hasNote}
|
||||
<span class="note-badge" title={t("transcript.has_note")}>📝</span>
|
||||
|
||||
Reference in New Issue
Block a user