Parakeet v Whisper h2h #26
@@ -84,7 +84,13 @@ mod cputime {
|
|||||||
fn normalize(s: &str) -> Vec<String> {
|
fn normalize(s: &str) -> Vec<String> {
|
||||||
s.to_lowercase()
|
s.to_lowercase()
|
||||||
.chars()
|
.chars()
|
||||||
.map(|c| if c.is_alphanumeric() || c == '\'' { c } else { ' ' })
|
.map(|c| {
|
||||||
|
if c.is_alphanumeric() || c == '\'' {
|
||||||
|
c
|
||||||
|
} else {
|
||||||
|
' '
|
||||||
|
}
|
||||||
|
})
|
||||||
.collect::<String>()
|
.collect::<String>()
|
||||||
.split_whitespace()
|
.split_whitespace()
|
||||||
.map(str::to_string)
|
.map(str::to_string)
|
||||||
@@ -121,6 +127,8 @@ fn self_test() {
|
|||||||
|
|
||||||
// ---- engines ----
|
// ---- engines ----
|
||||||
|
|
||||||
|
// ponytail: one Engine per process for its whole lifetime — variant size skew is irrelevant.
|
||||||
|
#[allow(clippy::large_enum_variant)]
|
||||||
enum Engine {
|
enum Engine {
|
||||||
Whisper(WhisperTranscriber),
|
Whisper(WhisperTranscriber),
|
||||||
Onnx(OnnxTranscriber),
|
Onnx(OnnxTranscriber),
|
||||||
@@ -282,12 +290,15 @@ fn run_corpus(contender: &str, engine: &Engine, load_ms: u128, corpus: &Path, ou
|
|||||||
(0usize, 0usize, 0u128, 0u64, 0f64);
|
(0usize, 0usize, 0u128, 0u64, 0f64);
|
||||||
for wav in &wavs {
|
for wav in &wavs {
|
||||||
let stem = wav.file_stem().unwrap().to_string_lossy().into_owned();
|
let stem = wav.file_stem().unwrap().to_string_lossy().into_owned();
|
||||||
let reference =
|
let reference = std::fs::read_to_string(wav.with_extension("txt")).unwrap_or_else(|e| {
|
||||||
std::fs::read_to_string(wav.with_extension("txt")).unwrap_or_else(|e| {
|
panic!(
|
||||||
panic!("missing reference {}: {e}", wav.with_extension("txt").display())
|
"missing reference {}: {e}",
|
||||||
});
|
wav.with_extension("txt").display()
|
||||||
let audio_secs =
|
)
|
||||||
read_wav_mono_16k(wav).map(|s| s.len() as f64 / SAMPLE_RATE as f64).unwrap_or(0.0);
|
});
|
||||||
|
let audio_secs = read_wav_mono_16k(wav)
|
||||||
|
.map(|s| s.len() as f64 / SAMPLE_RATE as f64)
|
||||||
|
.unwrap_or(0.0);
|
||||||
|
|
||||||
let cpu0 = cputime::process_cpu_ms();
|
let cpu0 = cputime::process_cpu_ms();
|
||||||
let t0 = Instant::now();
|
let t0 = Instant::now();
|
||||||
@@ -324,9 +335,18 @@ fn run_corpus(contender: &str, engine: &Engine, load_ms: u128, corpus: &Path, ou
|
|||||||
}
|
}
|
||||||
|
|
||||||
println!("---- {contender} summary ----");
|
println!("---- {contender} summary ----");
|
||||||
println!("files: {} audio: {audio_total:.1}s load: {load_ms}ms", wavs.len());
|
println!(
|
||||||
println!("WER: {:.2}% ({errors}/{words})", 100.0 * errors as f64 / words.max(1) as f64);
|
"files: {} audio: {audio_total:.1}s load: {load_ms}ms",
|
||||||
println!("RTF (wall): {:.3}", wall_total as f64 / 1000.0 / audio_total.max(0.001));
|
wavs.len()
|
||||||
|
);
|
||||||
|
println!(
|
||||||
|
"WER: {:.2}% ({errors}/{words})",
|
||||||
|
100.0 * errors as f64 / words.max(1) as f64
|
||||||
|
);
|
||||||
|
println!(
|
||||||
|
"RTF (wall): {:.3}",
|
||||||
|
wall_total as f64 / 1000.0 / audio_total.max(0.001)
|
||||||
|
);
|
||||||
println!(
|
println!(
|
||||||
"CPU-sec per audio-sec: {:.3}",
|
"CPU-sec per audio-sec: {:.3}",
|
||||||
cpu_total as f64 / 1000.0 / audio_total.max(0.001)
|
cpu_total as f64 / 1000.0 / audio_total.max(0.001)
|
||||||
@@ -349,7 +369,10 @@ fn run_live(contender: &str, engine: &Engine, wav: &Path) {
|
|||||||
}
|
}
|
||||||
latencies.sort_unstable();
|
latencies.sort_unstable();
|
||||||
let pct = |p: f64| latencies[(((latencies.len() - 1) as f64) * p) as usize];
|
let pct = |p: f64| latencies[(((latencies.len() - 1) as f64) * p) as usize];
|
||||||
println!("---- {contender} live ({WINDOW_SECS}s windows, n={}) ----", latencies.len());
|
println!(
|
||||||
|
"---- {contender} live ({WINDOW_SECS}s windows, n={}) ----",
|
||||||
|
latencies.len()
|
||||||
|
);
|
||||||
println!(
|
println!(
|
||||||
"p50={}ms p95={}ms max={}ms (budget: {}ms)",
|
"p50={}ms p95={}ms max={}ms (budget: {}ms)",
|
||||||
pct(0.50),
|
pct(0.50),
|
||||||
@@ -365,7 +388,9 @@ fn main() {
|
|||||||
self_test();
|
self_test();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let contender = args.first().expect("usage: asr_bench <contender> [--corpus DIR|--live WAV]");
|
let contender = args
|
||||||
|
.first()
|
||||||
|
.expect("usage: asr_bench <contender> [--corpus DIR|--live WAV]");
|
||||||
let flag = |name: &str| {
|
let flag = |name: &str| {
|
||||||
args.iter()
|
args.iter()
|
||||||
.position(|a| a == name)
|
.position(|a| a == name)
|
||||||
|
|||||||
Reference in new issue
Block a user