feat(hardware): enhance_notes, monitor_audio_level, stress_test_hardware commands
This commit is contained in:
1 parent
7d16c864e8
commit
6c343c4204
1 file changed
+309
@@ -1465,6 +1465,218 @@ pub async fn set_auto_start(app: AppHandle, enabled: bool) -> WaResult<()> {
|
||||
save_settings(&settings)
|
||||
}
|
||||
|
||||
/// Live level meter for a device, without starting a recording (FR-CAP-5): open
|
||||
/// the selected mic (`"input"`) or the render device in loopback (`"loopback"`)
|
||||
/// for a few seconds and stream `device://level` events so Settings ▸ Hardware
|
||||
/// can show whether audio is coming through. Reuses the normal capture path;
|
||||
/// discards frames (no transcription, no retained audio). One capture at a time,
|
||||
/// so it refuses while a recording is active.
|
||||
/// ponytail: transient monitor handle; ceiling = one monitor at a time.
|
||||
#[tauri::command]
|
||||
pub async fn monitor_audio_level(
|
||||
app: AppHandle,
|
||||
state: State<'_, AppState>,
|
||||
kind: String,
|
||||
device_id: Option<String>,
|
||||
duration_ms: Option<u64>,
|
||||
) -> WaResult<()> {
|
||||
use crate::audio::{AudioCapture, CaptureEvent, WasapiCapture};
|
||||
if state.session.lock().await.is_some() {
|
||||
return Err(WaError::new(
|
||||
"audio",
|
||||
"stop the current recording before testing a device",
|
||||
));
|
||||
}
|
||||
let duration =
|
||||
std::time::Duration::from_millis(duration_ms.unwrap_or(6000).clamp(1000, 20_000));
|
||||
|
||||
// Bounded channels so a slow consumer can't stall capture; frames are dropped.
|
||||
let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<Vec<f32>>(8);
|
||||
let (event_tx, event_rx) = std::sync::mpsc::sync_channel::<CaptureEvent>(64);
|
||||
std::thread::spawn(move || frame_rx.into_iter().for_each(drop));
|
||||
|
||||
let app_ev = app.clone();
|
||||
let kind_ev = kind.clone();
|
||||
std::thread::spawn(move || {
|
||||
for event in event_rx {
|
||||
if let CaptureEvent::Level(level) = event {
|
||||
let _ = app_ev.emit(
|
||||
"device://level",
|
||||
serde_json::json!({ "kind": kind_ev, "rms": level.rms, "peak": level.peak }),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// `start` (loopback) needs a WAV path; write to a temp file and delete it after.
|
||||
let tmp_wav = std::env::temp_dir().join(format!("wa-monitor-{}.wav", now_unix()));
|
||||
let handle = match kind.as_str() {
|
||||
"input" => WasapiCapture.start_microphone(device_id.as_deref(), frame_tx, event_tx),
|
||||
"loopback" => WasapiCapture.start(&tmp_wav, device_id.as_deref(), frame_tx, event_tx),
|
||||
_ => return Err(WaError::new("audio", "kind must be 'input' or 'loopback'")),
|
||||
}
|
||||
.map_err(|e| WaError::new("audio", e.to_string()))?;
|
||||
|
||||
tauri::async_runtime::spawn_blocking(move || {
|
||||
std::thread::sleep(duration);
|
||||
let _ = WasapiCapture.stop(handle);
|
||||
})
|
||||
.await
|
||||
.map_err(|e| WaError::new("audio", e.to_string()))?;
|
||||
|
||||
if kind == "loopback" {
|
||||
let _ = std::fs::remove_file(&tmp_wav);
|
||||
}
|
||||
let _ = app.emit(
|
||||
"device://level",
|
||||
serde_json::json!({ "kind": kind, "done": true }),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One row of the quick hardware stress test.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct StressResult {
|
||||
pub backend: String,
|
||||
pub model: String,
|
||||
pub rtf: f64,
|
||||
pub realtime: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct StressRecommendation {
|
||||
pub backend: String,
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct StressTestResult {
|
||||
pub results: Vec<StressResult>,
|
||||
pub recommended: Option<StressRecommendation>,
|
||||
}
|
||||
|
||||
/// Real-time recommendation: among rows that keep up with live speech
|
||||
/// (`rtf < 1`), prefer the **largest** model (most accurate), breaking ties by
|
||||
/// the **lowest** rtf (most headroom). `sizes` maps model id → size_mb (an
|
||||
/// accuracy proxy). `None` when nothing runs in real time.
|
||||
fn pick_realtime_recommendation(
|
||||
results: &[StressResult],
|
||||
sizes: &HashMap<String, u32>,
|
||||
) -> Option<StressRecommendation> {
|
||||
results
|
||||
.iter()
|
||||
.filter(|r| r.realtime)
|
||||
.max_by(|a, b| {
|
||||
let sa = sizes.get(&a.model).copied().unwrap_or(0);
|
||||
let sb = sizes.get(&b.model).copied().unwrap_or(0);
|
||||
sa.cmp(&sb)
|
||||
.then(b.rtf.partial_cmp(&a.rtf).unwrap_or(std::cmp::Ordering::Equal))
|
||||
})
|
||||
.map(|r| StressRecommendation {
|
||||
backend: r.backend.clone(),
|
||||
model: r.model.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Quick hardware stress test (FR-HW): benchmark each available backend against
|
||||
/// the installed whisper models on a fixed sample, measure real-time factor
|
||||
/// (elapsed / audio seconds), and recommend the most accurate model that still
|
||||
/// keeps up with live speech. Heavy (loads + runs each model) but explicit and
|
||||
/// progress-reported; runs off the async thread.
|
||||
/// ponytail: benchmarks installed models only, capped to 3 sizes.
|
||||
#[tauri::command]
|
||||
pub async fn stress_test_hardware(
|
||||
app: AppHandle,
|
||||
state: State<'_, AppState>,
|
||||
) -> WaResult<StressTestResult> {
|
||||
if state.session.lock().await.is_some() {
|
||||
return Err(WaError::new(
|
||||
"hardware",
|
||||
"stop the current recording before running the stress test",
|
||||
));
|
||||
}
|
||||
|
||||
// Fixed ~10s 16kHz mono sample. RTF timing is ~content-independent, so a
|
||||
// synthetic quiet tone is enough — no bundled speech clip needed.
|
||||
const SAMPLE_SECS: f64 = 10.0;
|
||||
let n = (16_000.0 * SAMPLE_SECS) as usize;
|
||||
let samples: Vec<f32> = (0..n).map(|i| (i as f32 * 0.05).sin() * 0.1).collect();
|
||||
let wav = std::env::temp_dir().join("wa-stress-sample.wav");
|
||||
crate::audio::write_wav_mono_16k(&wav, &samples)
|
||||
.map_err(|e| WaError::new("hardware", e.to_string()))?;
|
||||
|
||||
let backends: Vec<BackendId> = WinHardwareDetector
|
||||
.detect()
|
||||
.into_iter()
|
||||
.filter(|b| b.available)
|
||||
.map(|b| b.id)
|
||||
.collect();
|
||||
|
||||
// Installed models, smallest → largest, capped to bound runtime.
|
||||
let mut models: Vec<(String, u32)> = model_catalog::list("")
|
||||
.into_iter()
|
||||
.filter(|m| m.installed)
|
||||
.map(|m| (m.id, m.size_mb))
|
||||
.collect();
|
||||
models.sort_by_key(|(_, sz)| *sz);
|
||||
models.truncate(3);
|
||||
let sizes: HashMap<String, u32> = models.iter().cloned().collect();
|
||||
|
||||
if backends.is_empty() || models.is_empty() {
|
||||
let _ = std::fs::remove_file(&wav);
|
||||
return Err(WaError::new(
|
||||
"hardware",
|
||||
"no installed whisper model to benchmark — download one in Settings first",
|
||||
));
|
||||
}
|
||||
|
||||
let wav_for_task = wav.clone();
|
||||
let app_for_task = app.clone();
|
||||
let results = tauri::async_runtime::spawn_blocking(move || {
|
||||
let mut out: Vec<StressResult> = Vec::new();
|
||||
for backend in &backends {
|
||||
for (model_id, _size) in &models {
|
||||
let path = whisper_model_file(model_id);
|
||||
if !path.exists() {
|
||||
continue;
|
||||
}
|
||||
let _ = app_for_task.emit(
|
||||
"stress://progress",
|
||||
serde_json::json!({ "backend": backend.as_str(), "model": model_id }),
|
||||
);
|
||||
let start = std::time::Instant::now();
|
||||
let ok = match load_transcriber(*backend, &path, None) {
|
||||
Ok((t, _used)) => t.transcribe_file(&wav_for_task).is_ok(),
|
||||
Err(e) => {
|
||||
tracing::warn!("stress test load failed ({backend:?}/{model_id}): {e}");
|
||||
false
|
||||
}
|
||||
};
|
||||
if !ok {
|
||||
continue;
|
||||
}
|
||||
let rtf = start.elapsed().as_secs_f64() / SAMPLE_SECS;
|
||||
out.push(StressResult {
|
||||
backend: backend.as_str().to_string(),
|
||||
model: model_id.clone(),
|
||||
rtf,
|
||||
realtime: rtf < 1.0,
|
||||
});
|
||||
}
|
||||
}
|
||||
out
|
||||
})
|
||||
.await
|
||||
.map_err(|e| WaError::new("hardware", e.to_string()))?;
|
||||
|
||||
let _ = std::fs::remove_file(&wav);
|
||||
let recommended = pick_realtime_recommendation(&results, &sizes);
|
||||
Ok(StressTestResult {
|
||||
results,
|
||||
recommended,
|
||||
})
|
||||
}
|
||||
|
||||
/// True when the NPU ONNX Whisper model is downloaded (false on non-NPU builds).
|
||||
fn npu_model_installed() -> bool {
|
||||
#[cfg(feature = "npu")]
|
||||
@@ -3440,6 +3652,60 @@ pub async fn generate_tags(
|
||||
.map_err(|e| WaError::new("llm", e.to_string()))
|
||||
}
|
||||
|
||||
/// AI-enhance the user's rough notes into structured Markdown grounded in the
|
||||
/// transcript (Granola-style). Reuses the configured `LlmProvider` (local by
|
||||
/// default → no new egress). The caller passes the live editor buffer so we
|
||||
/// enhance exactly what the user sees, not a possibly-stale saved copy; we
|
||||
/// return the enhanced Markdown without persisting it — the UI decides to keep
|
||||
/// or undo it. Refuses a still-recording meeting; errors clearly with no
|
||||
/// provider configured. Invents nothing beyond the notes + transcript.
|
||||
#[tauri::command]
|
||||
pub async fn enhance_notes(
|
||||
state: State<'_, AppState>,
|
||||
meeting_id: MeetingId,
|
||||
notes: String,
|
||||
) -> WaResult<String> {
|
||||
let guard = state.session.lock().await;
|
||||
if guard.as_ref().is_some_and(|s| s.meeting_id == meeting_id) {
|
||||
return Err(WaError::new(
|
||||
"llm",
|
||||
"cannot enhance notes while this meeting is still recording — wait until it's stopped",
|
||||
));
|
||||
}
|
||||
drop(guard);
|
||||
|
||||
let settings = load_settings();
|
||||
let provider = llm_provider_from_settings(&settings).ok_or_else(|| {
|
||||
WaError::new(
|
||||
"llm",
|
||||
"no LLM provider is configured — enable one in Settings first",
|
||||
)
|
||||
})?;
|
||||
|
||||
let meeting = state
|
||||
.store
|
||||
.get_meeting(&meeting_id)
|
||||
.await
|
||||
.map_err(|e| WaError::new("storage", e.to_string()))?;
|
||||
let prompt = build_prompt(&meeting, None);
|
||||
|
||||
let system = "You expand a user's rough meeting notes into clear, well-structured Markdown. \
|
||||
Use ONLY facts stated in the transcript and the user's own notes — never invent details, names, \
|
||||
numbers, or decisions. Preserve the user's intent and any structure they started. Output Markdown \
|
||||
only, with no preamble or commentary.";
|
||||
let notes = notes.trim();
|
||||
let user = format!(
|
||||
"My rough notes:\n{}\n\nTranscript:\n{}",
|
||||
if notes.is_empty() { "(none yet)" } else { notes },
|
||||
prompt.transcript
|
||||
);
|
||||
provider
|
||||
.complete(system, &user)
|
||||
.await
|
||||
.map(|s| s.trim().to_string())
|
||||
.map_err(|e| WaError::new("llm", e.to_string()))
|
||||
}
|
||||
|
||||
// ---- Calendar / .pst (Phase 6) ----
|
||||
|
||||
/// Import events + attendees from a `.pst` (T6.1/T6.2, FR-CAL-1). The
|
||||
@@ -4929,6 +5195,49 @@ pub async fn privacy_self_check(state: State<'_, AppState>) -> WaResult<serde_js
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn row(backend: &str, model: &str, rtf: f64) -> StressResult {
|
||||
StressResult {
|
||||
backend: backend.into(),
|
||||
model: model.into(),
|
||||
rtf,
|
||||
realtime: rtf < 1.0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recommendation_picks_largest_realtime_model() {
|
||||
let sizes = HashMap::from([
|
||||
("tiny".to_string(), 32),
|
||||
("base".to_string(), 60),
|
||||
("small".to_string(), 190),
|
||||
]);
|
||||
let results = vec![
|
||||
row("cpu", "tiny", 0.4),
|
||||
row("cpu", "base", 0.9),
|
||||
row("cpu", "small", 1.4), // too slow — excluded
|
||||
row("vulkan", "small", 0.6),
|
||||
];
|
||||
let rec = pick_realtime_recommendation(&results, &sizes).unwrap();
|
||||
// small is the largest model that still runs in real time (on vulkan).
|
||||
assert_eq!(rec.model, "small");
|
||||
assert_eq!(rec.backend, "vulkan");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recommendation_breaks_size_ties_by_lowest_rtf() {
|
||||
let sizes = HashMap::from([("base".to_string(), 60)]);
|
||||
let results = vec![row("cpu", "base", 0.8), row("vulkan", "base", 0.3)];
|
||||
let rec = pick_realtime_recommendation(&results, &sizes).unwrap();
|
||||
assert_eq!(rec.backend, "vulkan"); // same model, more headroom
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recommendation_is_none_when_nothing_is_realtime() {
|
||||
let sizes = HashMap::from([("small".to_string(), 190)]);
|
||||
let results = vec![row("cpu", "small", 1.2)];
|
||||
assert!(pick_realtime_recommendation(&results, &sizes).is_none());
|
||||
}
|
||||
|
||||
/// Extracts the real hosted DirectML bundle (LZMA2+BCJ 7z) and checks the
|
||||
/// DLL lands flat. Skips if the binary isn't present (e.g. a lean checkout).
|
||||
#[cfg(feature = "npu")]
|
||||
|
||||
Reference in new issue
Block a user