ai: call transcripts (each side recorded, merged by time) and summaries; search on Recordings
ci / check (push) Successful in 1m22s
ci / images (push) Successful in 27s

This commit is contained in:
iamdoubz committed 2026-10-21 19:10:02 -05:00
1 parent d9f9c6e444
commit 4d331fb70e
56 files changed
+5186 -37

No files matched your search

+89
View File
@@ -0,0 +1,89 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import { describe, expect, it } from 'vitest';
import { mergeSides, sideFiles, speechStretches, transcriptText, wavSlice } from './calls.ts';
describe('call transcripts', () => {
it('side files sit next to the recording', () => {
expect(sideFiles('2026/10/21/1792.5.wav')).toEqual([
'2026/10/21/1792.5-in.wav',
'2026/10/21/1792.5-out.wav',
]);
});
it('merges by time; the caller first when both start together', () => {
expect(
mergeSides(
[
{ at: 0, text: 'Hello?' },
{ at: 5, text: 'Bye.' },
],
[
{ at: 0, text: 'Hi.' },
{ at: 2.5, text: 'How can I help?' },
],
).map((l) => `${l.side} ${l.at}`),
).toEqual(['caller 0', 'other 0', 'other 2.5', 'caller 5']);
});
it('labels the lines and cuts a long call', () => {
const lines = mergeSides([{ at: 0, text: 'a'.repeat(50) }], [{ at: 1, text: 'b' }]);
expect(transcriptText(lines)).toBe(`Caller: ${'a'.repeat(50)}\nOther side: b`);
expect(transcriptText(lines, 20)).toBe(
`Caller: ${'a'.repeat(12)}\n[the rest of the call is cut]`,
);
});
});
describe('speech stretches', () => {
/** 8 kHz mono 16-bit: [seconds, loud?] pieces one after another. */
const wav = (parts: [number, boolean][]) => {
const samples = parts.flatMap(([s, loud]) =>
Array.from({ length: s * 8000 }, (_, i) => (loud ? Math.round(5000 * Math.sin(i / 3)) : 0)),
);
const b = Buffer.alloc(44 + samples.length * 2);
b.writeUInt32LE(8000, 24);
b.writeUInt32LE(16000, 28);
samples.forEach((v, i) => b.writeInt16LE(v, 44 + i * 2));
return b;
};
const round = (x: { from: number; to: number }[]) =>
x.map((s) => [Math.round(s.from * 10) / 10, Math.round(s.to * 10) / 10]);
it('finds each stretch of speech, joining short pauses, with a little margin', () => {
const w = wav([
[1, false],
[2, true],
[0.4, false], // a breath: same stretch
[1, true],
[3, false],
[1, true],
[1, false],
]);
expect(round(speechStretches(w))).toEqual([
[0.8, 4.6],
[7.2, 8.6],
]);
});
it('silence and a click are nothing', () => {
expect(speechStretches(wav([[3, false]]))).toEqual([]);
expect(
speechStretches(
wav([
[1, false],
[0.1, true],
[1, false],
]),
),
).toEqual([]);
});
it('a slice is a WAV of just those seconds', () => {
const w = wav([[2, true]]);
const s = wavSlice(w, 0.5, 1.5);
expect(s.length).toBe(44 + 16000);
expect(s.readUInt32LE(40)).toBe(16000);
expect(s.readUInt32LE(4)).toBe(36 + 16000);
expect(s.subarray(44, 54)).toEqual(w.subarray(44 + 8000, 54 + 8000));
});
});
+177
View File
@@ -0,0 +1,177 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// Call transcripts and summaries (spec 10 §3). With ai.callTranscripts on, sp-sub-record also
// writes <call>-in.wav (what the caller said) and <call>-out.wav (what they heard), so who said
// what needs no speaker detection. A job writes each side out, merges them by time, asks ai.llm
// for a summary (ai.callSummaries), and deletes the two side files: the mixed recording stays.
import { access, readFile, rm } from 'node:fs/promises';
import { join } from 'node:path';
import { and, asc, eq, gt, isNull, lt, sql } from 'drizzle-orm';
import type { Db } from '../db/client.ts';
import { recordings } from '../db/schema.ts';
import { getSetting } from '../services/settings.ts';
import { AiError, chat, transcribeSegments } from './client.ts';
export type Line = { side: 'caller' | 'other'; at: number; text: string };
/** The two side files next to a recording (relative, like recordings.file_path). */
export const sideFiles = (filePath: string) => {
const base = filePath.replace(/\.wav$/, '');
return [`${base}-in.wav`, `${base}-out.wav`] as const;
};
/**
* Where someone speaks in a 16-bit mono WAV (MixMonitor's: 44-byte header), in seconds: 20 ms
* frames louder than the line's noise, joined across pauses under `gap` s. Whisper's own timings
* drift across the long silences one side of a call has while the other talks, so each stretch is
* transcribed on its own, from its known start.
*/
export function speechStretches(wav: Buffer, gap = 0.6): { from: number; to: number }[] {
const rate = wav.readUInt32LE(24) || 8000;
const frame = Math.round(rate / 50) * 2; // bytes in 20 ms
const levels: number[] = [];
for (let i = 44; i + frame <= wav.length; i += frame) {
let sum = 0;
for (let j = i; j < i + frame; j += 2) sum += Math.abs(wav.readInt16LE(j));
levels.push(sum / (frame / 2));
}
// ponytail: a fixed share of the loudest speech, with a floor for silent lines; tune if a noisy
// line splits badly.
const loud = [...levels].sort((a, b) => a - b)[Math.floor(levels.length * 0.95)] ?? 0;
const threshold = Math.max(120, loud / 10);
const out: { from: number; to: number }[] = [];
levels.forEach((l, i) => {
if (l < threshold) return;
const at = i * 0.02;
const last = out.at(-1);
if (last && at - last.to < gap) last.to = at + 0.02;
else out.push({ from: at, to: at + 0.02 });
});
const end = levels.length * 0.02;
return out
.filter((s) => s.to - s.from >= 0.2) // a click, not a word
.map((s) => ({ from: Math.max(0, s.from - 0.2), to: Math.min(end, s.to + 0.2) }));
}
/** Seconds `from`–`to` of a WAV as a WAV of its own. */
export function wavSlice(wav: Buffer, from: number, to: number): Buffer {
const bytesPerSec = wav.readUInt32LE(28) || 16000;
const at = (s: number) => 44 + Math.min(wav.length - 44, Math.round(s * bytesPerSec) & ~1);
const data = wav.subarray(at(from), at(to));
const head = Buffer.from(wav.subarray(0, 44));
head.writeUInt32LE(36 + data.length, 4);
head.writeUInt32LE(data.length, 40);
return Buffer.concat([head, data]);
}
/** One side of a call, stretch by stretch, each line at its time in the whole call. */
async function transcribeSide(stt: Parameters<typeof transcribeSegments>[0], wav: Buffer) {
const lines: Omit<Line, 'side'>[] = [];
for (const s of speechStretches(wav))
for (const seg of await transcribeSegments(stt, wavSlice(wav, s.from, s.to)))
lines.push({ at: Math.round((s.from + seg.at) * 10) / 10, text: seg.text });
return lines;
}
/** Both sides in time order (the files start together, so their timings line up). */
export const mergeSides = (caller: Omit<Line, 'side'>[], other: Omit<Line, 'side'>[]): Line[] =>
[
...caller.map((l) => ({ side: 'caller' as const, ...l })),
...other.map((l) => ({ side: 'other' as const, ...l })),
].sort((a, b) => a.at - b.at || (a.side === 'caller' ? -1 : 1));
/** "Caller: …" lines, cut to fit a small model's context. */
export function transcriptText(lines: Line[], max = 12_000): string {
const text = lines
.map((l) => `${l.side === 'caller' ? 'Caller' : 'Other side'}: ${l.text}`)
.join('\n');
// ponytail: a long call is cut, not chunked; summarize in parts if calls this long matter.
return text.length > max ? `${text.slice(0, max)}\n[the rest of the call is cut]` : text;
}
const SUMMARY_PROMPT = `You summarize business phone calls from their transcript. Write in the \
language of the call, in plain text, no markdown headings:
- 1 to 3 sentences on what the call was about and what was agreed;
- then a line "Action items:" and one "- " line per item (who does what), or "Action items: none".
Use only what the transcript says. The transcript is automatic and may have mistakes.`;
/** Recordings the job still has to do: recent, finished a moment ago, not tried yet. */
const pending = (db: Db) =>
db
.select({ id: recordings.id, filePath: recordings.filePath })
.from(recordings)
.where(
and(
isNull(recordings.aiDoneAt),
gt(recordings.startedAt, sql`now() - interval '2 days'`),
lt(recordings.createdAt, sql`now() - interval '20 seconds'`), // files closed
),
)
.orderBy(asc(recordings.startedAt))
.limit(20);
const exists = (f: string) =>
access(f).then(
() => true,
() => false,
);
/** One run: every pending recording, one at a time (the GPU is shared). */
export async function transcribeCalls(d: {
db: Db;
dir: string;
log: { error(o: object, m: string): void };
}): Promise<number> {
const stt = await getSetting(d.db, 'ai.stt');
if (!stt || !(await getSetting(d.db, 'ai.callTranscripts'))) return 0;
const llm = (await getSetting(d.db, 'ai.callSummaries'))
? await getSetting(d.db, 'ai.llm')
: null;
let done = 0;
for (const r of await pending(d.db)) {
const files = sideFiles(r.filePath).map((f) => join(d.dir, f));
const finish = (set: Partial<typeof recordings.$inferInsert>) =>
d.db
.update(recordings)
.set({ aiDoneAt: new Date(), ...set })
.where(eq(recordings.id, r.id));
if (!(await exists(files[0]!)) || !(await exists(files[1]!))) {
await finish({ aiError: 'Recorded before call transcripts were on' });
continue;
}
let lines: Line[];
try {
const [caller, other] = [
await transcribeSide(stt, await readFile(files[0]!)),
await transcribeSide(stt, await readFile(files[1]!)),
];
lines = mergeSides(caller, other);
} catch (err) {
// The AI stack being down or busy: try again on the next run. Anything else: give up.
if (err instanceof AiError && /Can't reach|didn't answer/.test(err.message)) {
d.log.error({ err, recording: r.id }, 'call transcript: speech to text unreachable');
return done;
}
await finish({ aiError: err instanceof Error ? err.message : String(err) });
continue;
}
let summary: string | null = null;
let aiError: string | null = null;
if (llm && lines.length)
try {
summary = await chat(
llm,
[
{ role: 'system', content: SUMMARY_PROMPT },
{ role: 'user', content: transcriptText(lines) },
],
{ maxTokens: 2000, timeoutMs: 300_000 },
);
} catch (err) {
aiError = `Summary: ${err instanceof Error ? err.message : String(err)}`;
}
await finish({ transcript: lines, summary, aiError });
for (const f of files) await rm(f, { force: true });
done++;
}
return done;
}
+18
View File
@@ -80,6 +80,24 @@ export async function transcribe(ep: AiEndpoint, audio: Buffer, timeoutMs = 180_
return text.trim();
}
/** Speech to text with timings (verbose_json): what was said, and from which second. */
export async function transcribeSegments(ep: AiEndpoint, audio: Buffer, timeoutMs = 600_000) {
const form = new FormData();
form.append('file', new Blob([new Uint8Array(audio)], { type: 'audio/wav' }), 'audio.wav');
form.append('model', ep.model);
form.append('response_format', 'verbose_json');
const r = await call(ep.url, 'audio/transcriptions', { method: 'POST', body: form }, timeoutMs);
const j = (await r.json()) as {
text?: unknown;
segments?: { start?: unknown; text?: unknown }[];
};
// A server without segments: the whole text, from the start.
const segs = Array.isArray(j.segments)
? j.segments.map((s) => ({ at: Number(s.start) || 0, text: String(s.text ?? '').trim() }))
: [{ at: 0, text: typeof j.text === 'string' ? j.text.trim() : '' }];
return segs.filter((s) => s.text);
}
/** Text to speech: a WAV. */
export async function speak(ep: AiTtsEndpoint, input: string, timeoutMs = 60_000) {
const r = await call(
@@ -0,0 +1,4 @@
ALTER TABLE "recordings" ADD COLUMN "transcript" jsonb;--> statement-breakpoint
ALTER TABLE "recordings" ADD COLUMN "summary" text;--> statement-breakpoint
ALTER TABLE "recordings" ADD COLUMN "ai_done_at" timestamp with time zone;--> statement-breakpoint
ALTER TABLE "recordings" ADD COLUMN "ai_error" text;
File diff suppressed because it is too large. Load diff
@@ -50,6 +50,13 @@
"when": 1792624567407,
"tag": "0006_voicemail_transcript",
"breakpoints": true
},
{
"idx": 7,
"version": "7",
"when": 1792626762332,
"tag": "0007_recording_transcripts",
"breakpoints": true
}
]
}
+6
View File
@@ -533,6 +533,12 @@ export const recordings = pgTable('recordings', {
startedAt: tstz().notNull(),
durationMs: integer(),
sizeBytes: bigint({ mode: 'number' }),
/** Spec 10 §3: [{ side: 'caller' | 'other', at: seconds, text }], in time order. */
transcript: jsonb().$type<{ side: 'caller' | 'other'; at: number; text: string }[]>(),
summary: text(),
/** When the transcript job finished with it (null: not yet); aiError: why it has none. */
aiDoneAt: tstz(),
aiError: text(),
...timestamps,
});
@@ -2,4 +2,5 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_101=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,3 +2,4 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,5 +2,6 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_101=2
SP_TCADMIN_101=1
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,6 +2,7 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_103=2
SP_CLASS_104=2
SP_CLASS_105=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,6 +2,7 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_101=2
SP_CLASS_102=2
SP_CLASS_103=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,5 +2,6 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_101=2
SP_CLASS_102=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,5 +2,6 @@
SP_DEFAULT_CID=13125551234
SP_DEFAULT_E911_CID=13125551234
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_101=2
SP_TCADMIN_101=1
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,6 +2,7 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_99=2
SP_CLASS_101=2
SP_CLASS_102=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,4 +2,5 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_101=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,6 +2,7 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_101=2
SP_CLASS_102=2
SP_CLASS_103=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,6 +2,7 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_101=2
SP_CLASS_102=2
SP_CLASS_103=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,5 +2,6 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_1=2
SP_CLASS_12=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,3 +2,4 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,4 +2,5 @@
SP_DEFAULT_CID=13125551234
SP_DEFAULT_E911_CID=13125551234
SP_REC_ANNOUNCE=custom/44444444-4444-4444-8444-444444444448
SP_REC_SIDES=
SP_CLASS_101=2
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
@@ -2,5 +2,6 @@
SP_DEFAULT_CID=
SP_DEFAULT_E911_CID=
SP_REC_ANNOUNCE=
SP_REC_SIDES=
SP_CLASS_99=0
SP_CLASS_102=1
@@ -83,11 +83,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[${IF($["${SP_FWDHOPS}" = ""]?0:${SP_FWDHOPS})} +
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=${STRFTIME(${EPOCH},,%Y/%m/%d)}/${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(${SP_REC},br(${SP_REC:0:-4}-in.wav)t(${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
+2
View File
@@ -73,6 +73,8 @@ export function renderGlobals(snap: Snapshot): string {
`SP_DEFAULT_CID=${defaultCid}`,
`SP_DEFAULT_E911_CID=${didNumber(st['emergency.defaultCallerIdDidId']) || defaultCid}`,
`SP_REC_ANNOUNCE=${snap.recordingAnnouncement ? s.sound(snap.recordingAnnouncement) : ''}`,
// Spec 10 §3: also record each side on its own, for the transcript job.
`SP_REC_SIDES=${st['ai.callTranscripts'] && st['ai.stt'] ? '1' : ''}`,
];
const exts = [...snap.extensions].sort(byNumber);
for (const e of exts) lines.push(`SP_CLASS_${s.digits(e.number)}=${CLASS_RANK[e.outboundClass]}`);
@@ -86,11 +86,16 @@ exten => s,1,Set(__SP_FWDHOPS=$[\${IF($["\${SP_FWDHOPS}" = ""]?0:\${SP_FWDHOPS})
same => n(stop),Return()
; ─── Start recording this call once. File: YYYY/MM/DD/<uniqueid>.wav under the monitor dir.
; This runs on the calling side, so with SP_REC_SIDES (call transcripts on) <uniqueid>-in.wav is
; what the caller said and -out.wav what they heard (the other side).
[sp-sub-record]
exten => s,1,GotoIf($["\${SP_REC}" != ""]?done)
same => n,Set(__SP_REC=\${STRFTIME(\${EPOCH},,%Y/%m/%d)}/\${UNIQUEID}.wav)
same => n,Set(CDR(SP_REC)=\${SP_REC}) ; cdr_manager maps CDR variables only
same => n,GotoIf($["\${GLOBAL(SP_REC_SIDES)}" = "1"]?sides)
same => n,MixMonitor(\${SP_REC},b)
same => n,Return()
same => n(sides),MixMonitor(\${SP_REC},br(\${SP_REC:0:-4}-in.wav)t(\${SP_REC:0:-4}-out.wav))
same => n(done),Return()
; ─── Auto-answer headers for paging/intercom (Yealink, Grandstream, Polycom, Snom)
+4
View File
@@ -20,6 +20,7 @@ import type { Generator } from './generator/index.ts';
import { todayIn } from './generator/load.ts';
import type { LiveState } from './live/state.ts';
import { cleanupRecordings } from './routes/recordings.ts';
import { transcribeCalls } from './ai/calls.ts';
import { audit } from './services/audit.ts';
import { backupDue, runBackup } from './services/backup.ts';
import { logEvent } from './services/events.ts';
@@ -258,6 +259,9 @@ export function startJobs(d: JobDeps): () => void {
run('trunkWatch', j.trunkWatch);
if (minute % 5 === 0) run('publicIpWatch', j.publicIpWatch);
if (minute % 60 === 0) run('diskWatch', j.diskWatch);
run('callTranscripts', async () => {
await transcribeCalls({ db: d.db, dir: resolve(d.config.RECORDINGS_DIR), log: d.log });
});
minute++;
};
tick();
+57 -18
View File
@@ -3,18 +3,28 @@
// routes. Admins see all; a user may play the recordings of calls they took part in.
import { rm } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { and, desc, eq, inArray, lt, sql } from 'drizzle-orm';
import { and, desc, eq, ilike, inArray, lt, or, sql } from 'drizzle-orm';
import type { FastifyInstance } from 'fastify';
import { z } from 'zod';
import type { ListResponse, RecordingDto } from '@selfphoneng/shared';
import type { ListResponse, RecordingDto, TranscriptDto } from '@selfphoneng/shared';
import { requireAdmin, requireAuth } from '../auth/guards.ts';
import type { Db } from '../db/client.ts';
import { callExtensions, calls, extensions, recordings } from '../db/schema.ts';
import { badRequest, forbidden, notFound } from '../errors.ts';
import { audit } from '../services/audit.ts';
import { sideFiles } from '../ai/calls.ts';
/** The recording and, when call transcripts left them, its two side files. */
const removeFiles = async (dir: string, filePath: string) => {
for (const f of [filePath, ...sideFiles(filePath)]) await rm(join(dir, f), { force: true });
};
const IdParams = z.object({ id: z.uuid() });
const ListQuery = z.object({ cursor: z.string().max(100).optional() });
const ListQuery = z.object({
cursor: z.string().max(100).optional(),
/** Words in the transcript or summary (spec 10 §3). */
q: z.string().trim().max(100).optional(),
});
/** Deletes recordings older than `days` (file and row); 0 keeps them forever (04 §8). */
export async function cleanupRecordings(db: Db, dir: string, days: number): Promise<number> {
@@ -23,7 +33,7 @@ export async function cleanupRecordings(db: Db, dir: string, days: number): Prom
.select({ id: recordings.id, filePath: recordings.filePath })
.from(recordings)
.where(lt(recordings.startedAt, sql`now() - make_interval(days => ${days})`));
for (const r of old) await rm(join(dir, r.filePath), { force: true });
for (const r of old) await removeFiles(dir, r.filePath);
if (old.length)
await db.delete(recordings).where(
inArray(
@@ -34,6 +44,9 @@ export async function cleanupRecordings(db: Db, dir: string, days: number): Prom
return old.length;
}
/** A search term taken literally by ILIKE: \, % and _ escaped. */
const likeEscape = (s: string) => s.replace(/[\\%_]/g, (c) => `\\${c}`);
export async function recordingRoutes(app: FastifyInstance) {
const { db } = app;
const dir = resolve(app.config.RECORDINGS_DIR);
@@ -48,14 +61,24 @@ export async function recordingRoutes(app: FastifyInstance) {
'/api/recordings',
{ preHandler: requireAdmin },
async (req): Promise<ListResponse<RecordingDto>> => {
const { cursor } = ListQuery.parse(req.query);
const { cursor, q } = ListQuery.parse(req.query);
const before = cursor ? new Date(Buffer.from(cursor, 'base64url').toString()) : null;
if (before && Number.isNaN(before.getTime())) throw badRequest('Bad cursor');
const rows = await db
.select({ r: recordings, c: calls })
.from(recordings)
.leftJoin(calls, eq(calls.recordingId, recordings.id))
.where(before ? lt(recordings.startedAt, before) : undefined)
.where(
and(
before ? lt(recordings.startedAt, before) : undefined,
q
? or(
ilike(sql`${recordings.transcript}::text`, `%${likeEscape(q)}%`),
ilike(recordings.summary, `%${likeEscape(q)}%`),
)
: undefined,
),
)
.orderBy(desc(recordings.startedAt))
.limit(51);
const items = rows.slice(0, 50);
@@ -65,6 +88,9 @@ export async function recordingRoutes(app: FastifyInstance) {
startedAt: r.startedAt.toISOString(),
durationMs: r.durationMs,
sizeBytes: r.sizeBytes,
summary: r.summary,
hasTranscript: r.transcript !== null,
aiError: r.aiError,
call: c
? {
id: c.id,
@@ -83,19 +109,32 @@ export async function recordingRoutes(app: FastifyInstance) {
},
);
/** Admins, or a participant: their extension is on the call this recording belongs to. */
async function mayHear(user: { id: string; role: string }, recordingId: string) {
if (user.role === 'admin') return;
const [mine] = await db
.select({ id: calls.id })
.from(calls)
.innerJoin(callExtensions, eq(callExtensions.callId, calls.id))
.innerJoin(extensions, eq(extensions.id, callExtensions.extensionId))
.where(and(eq(calls.recordingId, recordingId), eq(extensions.userId, user.id)))
.limit(1);
if (!mine) throw forbidden('Not your call');
}
app.get('/api/recordings/:id/transcript', { preHandler: requireAuth }, async (req) => {
const r = await getRecording(IdParams.parse(req.params).id);
await mayHear(req.user!, r.id);
return {
lines: r.transcript ?? [],
summary: r.summary,
aiError: r.aiError,
} satisfies TranscriptDto;
});
app.get('/api/recordings/:id/audio', { preHandler: requireAuth }, async (req, reply) => {
const r = await getRecording(IdParams.parse(req.params).id);
if (req.user!.role !== 'admin') {
// Only a participant: their extension is on the call this recording belongs to.
const [mine] = await db
.select({ id: calls.id })
.from(calls)
.innerJoin(callExtensions, eq(callExtensions.callId, calls.id))
.innerJoin(extensions, eq(extensions.id, callExtensions.extensionId))
.where(and(eq(calls.recordingId, r.id), eq(extensions.userId, req.user!.id)))
.limit(1);
if (!mine) throw forbidden('Not your call');
}
await mayHear(req.user!, r.id);
if ((req.query as { download?: string }).download)
reply.header(
'content-disposition',
@@ -106,7 +145,7 @@ export async function recordingRoutes(app: FastifyInstance) {
app.delete('/api/recordings/:id', { preHandler: requireAdmin }, async (req) => {
const r = await getRecording(IdParams.parse(req.params).id);
await rm(join(dir, r.filePath), { force: true });
await removeFiles(dir, r.filePath);
await db.delete(recordings).where(eq(recordings.id, r.id)); // calls.recording_id → null
await audit(db, {
actorUserId: req.user!.id,
+149
View File
@@ -0,0 +1,149 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// Call transcripts (spec 10 §3): with ai.callTranscripts on, sp-sub-record writes the mixed
// recording plus <call>-in.wav (what the caller said) and <call>-out.wav (what they heard). A Local
// channel records itself through sp-sub-record and dials one that answers and stays silent; the
// other half plays digits as "the caller". So -in must have sound and -out (nearly) none.
// Usage: node --conditions=source apps/api/test/asterisk/rec-test.ts (ASTERISK_IMAGE)
import { spawnSync } from 'node:child_process';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { SettingsSchemas, type SettingKey } from '@selfphoneng/shared';
import { GENERATED_FILES, render } from '../../src/generator/index.ts';
import type { Snapshot } from '../../src/generator/load.ts';
const IMAGE = process.env.ASTERISK_IMAGE ?? 'selfphoneng-asterisk';
const AST = 'sp-test-rec';
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
const docker = (...args: string[]) => {
const r = spawnSync('docker', args, { encoding: 'utf8' });
return { ok: r.status === 0, out: `${r.stdout ?? ''}${r.stderr ?? ''}` };
};
const cli = (cmd: string) => docker('exec', AST, 'asterisk', '-rx', cmd).out;
const log = (...a: unknown[]) => console.log('[rec-test]', ...a);
function snapshot(): Snapshot {
const settings = Object.fromEntries(
(Object.keys(SettingsSchemas) as SettingKey[])
.filter((k) => k !== 'sms.webhookToken')
.map((k) => [k, SettingsSchemas[k].parse(undefined)]),
) as Snapshot['settings'];
settings['ai.callTranscripts'] = true;
settings['ai.stt'] = { url: 'http://127.0.0.1:8000/v1', model: 'whisper' };
return {
today: '2026-10-21',
settings,
dids: [],
trunks: [],
routes: [],
ringGroups: [],
queues: [],
conferences: [],
pagingGroups: [],
ivrs: [],
timeConditions: [],
holidays: [],
announcements: [],
recordingAnnouncement: null,
extensions: [],
mohClasses: [],
};
}
async function main() {
const dir = mkdtempSync(join(tmpdir(), 'sp-rec-'));
const conf = join(dir, 'conf');
mkdirSync(conf);
const files = render(snapshot());
for (const f of GENERATED_FILES) writeFileSync(join(conf, f), files[f]);
const problems: string[] = [];
try {
docker('rm', '-f', AST);
const run = docker(
'run',
'-d',
'--rm',
'--name',
AST,
'-e',
'AMI_SECRET=rec-test-secret',
'-v',
`${conf}:/etc/asterisk/selfphoneng`,
IMAGE,
);
if (!run.ok) throw new Error(`asterisk: ${run.out}`);
for (
let i = 0;
i < 60 && !docker('exec', AST, 'asterisk', '-rx', 'core waitfullybooted').ok;
i++
)
await sleep(500);
log(
cli('dialplan show sp-sub-record')
.split('\n')
.filter((l) => l.includes('MixMonitor'))
.join(' | '),
);
for (const line of [
'dialplan add extension rec,1,Gosub,"sp-sub-record,s,1" into sp-rectest',
'dialplan add extension rec,2,Dial,"Local/play@sp-rectest/n" into sp-rectest',
'dialplan add extension play,1,Answer, into sp-rectest',
'dialplan add extension play,2,Wait,4 into sp-rectest',
])
cli(line);
cli('channel originate Local/rec@sp-rectest/n application Playback digits/1&digits/2&digits/3');
await sleep(6000);
const ls = docker(
'exec',
AST,
'sh',
'-c',
'find /var/spool/asterisk/monitor -name "*.wav" -exec stat -c "%s %n" {} \\;',
).out.trim();
log(ls.replaceAll('\n', ' | ') || 'no files');
const sized = ls
.split('\n')
.filter(Boolean)
.map((l) => {
const [size, name] = l.split(' ');
return { size: Number(size), name: name! };
});
const mixed = sized.find((f) => /\/\d+\.\d+\.wav$/.test(f.name));
if (!mixed) problems.push('no mixed recording');
else {
for (const side of ['-in.wav', '-out.wav']) {
const f = sized.find((x) => x.name === mixed.name.replace(/\.wav$/, side));
if (!f) problems.push(`no ${side} next to ${mixed.name}`);
else if (f.size < 8000) problems.push(`${side} has no audio (${f.size} bytes)`);
}
// Each side on its own, the right way round: the caller's file has the digits, the silent
// side's has nothing (a copy of the mix would have them too).
const loud = (side: string) => {
const wav = spawnSync('docker', ['exec', AST, 'cat', mixed.name.replace(/\.wav$/, side)])
.stdout as Buffer;
let sum = 0;
for (let i = 44; i + 1 < wav.length; i += 2) sum += Math.abs(wav.readInt16LE(i));
return Math.round(sum / ((wav.length - 44) / 2));
};
const [mix, caller, other] = [loud('.wav'), loud('-in.wav'), loud('-out.wav')];
log(`average level: mix ${mix}, -in ${caller}, -out ${other}`);
if (!(mix > 200 && caller > 200 && other < caller / 10))
problems.push(
`-in should be the caller (loud), -out the silent side: ${caller} / ${other}`,
);
}
} catch (e) {
problems.push((e as Error).message);
} finally {
docker('rm', '-f', AST);
rmSync(dir, { recursive: true, force: true });
}
if (problems.length) {
console.log('FAIL call recording sides');
for (const p of problems) console.log(` ${p}`);
process.exit(1);
}
console.log('ok call recording sides (mixed, -in and -out with audio)');
}
await main();
+9
View File
@@ -22,3 +22,12 @@ export function silenceWav(seconds = 1) {
h.writeUInt32LE(data.length, 40);
return Buffer.concat([h, data]);
}
/** `silence` s of silence, then `seconds` s of a 440 Hz tone, as an 8 kHz mono 16-bit wav. */
export function toneWav(seconds = 1, silence = 0) {
const w = silenceWav(seconds + silence);
const start = 44 + Math.round(silence * 8000) * 2;
for (let i = 0; start + i * 2 < w.length; i++)
w.writeInt16LE(Math.round(6000 * Math.sin((2 * Math.PI * 440 * i) / 8000)), start + i * 2);
return w;
}
+203
View File
@@ -0,0 +1,203 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// Spec 10 §3: call transcripts and summaries. The job writes out each side of a recorded call,
// merges them by time, asks the language model for a summary and removes the side files; the
// Recordings page can search them, and only the people on the call (and admins) can read them.
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { createServer, type Server } from 'node:http';
import type { AddressInfo } from 'node:net';
import { join } from 'node:path';
import { eq, sql } from 'drizzle-orm';
import type { LightMyRequestResponse } from 'fastify';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type { AmiMessage } from '../src/ami/events.ts';
import { transcribeCalls } from '../src/ai/calls.ts';
import { encrypt } from '../src/crypto/secrets.ts';
import { cdr, extensions, recordings } from '../src/db/schema.ts';
import { aggregateCall, toCdrRow } from '../src/services/cdr.ts';
import { setSetting } from '../src/services/settings.ts';
import { silenceWav, toneWav } from './audio.ts';
import { hasTestDb } from './db.ts';
import { ADMIN, H, createTestApp } from './app.ts';
const FILE = '2026/10/21/1792351748.9.wav';
describe.skipIf(!hasTestDb)('call transcripts and summaries', () => {
let t: Awaited<ReturnType<typeof createTestApp>>;
let ai: Server;
let down = false;
const asked: string[] = [];
let as: Record<'admin' | 'alice' | 'bob', (url: string) => Promise<LightMyRequestResponse>>;
let recId: string;
const dir = () => t.app.config.RECORDINGS_DIR;
const log = { error: () => {} };
const sides = () => [`${FILE.slice(0, -4)}-in.wav`, `${FILE.slice(0, -4)}-out.wav`];
/** The side files: someone speaking (a tone) on each; the other side speaks longer. */
const writeSides = () => {
writeFileSync(join(dir(), sides()[0]!), toneWav(1));
writeFileSync(join(dir(), sides()[1]!), toneWav(2));
};
const pending = () =>
t.db
.update(recordings)
.set({ aiDoneAt: null, aiError: null, transcript: null, summary: null })
.where(eq(recordings.id, recId));
beforeAll(async () => {
ai = createServer((req, res) => {
let body = '';
req.setEncoding('latin1').on('data', (c: string) => (body += c));
req.on('end', () => {
if (down) return res.destroy();
asked.push(req.url!);
if (req.url === '/v1/audio/transcriptions')
return res.end(
JSON.stringify(
body.length < 30_000 // the caller's stretch: about a second
? {
text: '…',
segments: [
{ start: 0.4, text: ' Hi, I never got the invoice.' },
{ start: 6.2, text: ' Great, thanks.' },
],
}
: { text: '…', segments: [{ start: 2.9, text: " I'll email it today." }] },
),
);
if (req.url === '/v1/chat/completions') {
expect(body).toContain('Caller: Hi, I never got the invoice.\\nOther side: I');
return res.end(
JSON.stringify({
choices: [
{
message: {
content:
'<think>short call</think>The caller never got their invoice; it will be emailed today.\nAction items:\n- Email the invoice today',
},
},
],
}),
);
}
res.writeHead(404).end();
});
});
await new Promise<void>((r) => ai.listen(0, '127.0.0.1', r));
const url = `http://127.0.0.1:${(ai.address() as AddressInfo).port}/v1`;
t = await createTestApp();
await t.clearMustChange(ADMIN.email);
const alice = await t.addUser('alice@example.com', 'alice-password-123');
await t.addUser('bob@example.com', 'bob-password-123');
await t.db.insert(extensions).values({
number: '101',
name: 'Alice',
userId: alice.id,
sipPasswordEnc: encrypt('x'),
voicemailPinEnc: encrypt('4826'),
});
const events: AmiMessage[] = JSON.parse(
readFileSync(new URL('./fixtures/cdr/inbound-user.json', import.meta.url), 'utf8'),
);
for (const e of events) await t.db.insert(cdr).values(toCdrRow({ ...e, sp_recording: FILE }));
mkdirSync(join(dir(), '2026/10/21'), { recursive: true });
writeFileSync(join(dir(), FILE), silenceWav(2));
await aggregateCall(t.db, String(events[0]!.LinkedID), { recordingsDir: dir() });
const [r] = await t.db.select().from(recordings);
recId = r!.id;
// A call that ended a minute ago (the fixture's date is older than the job looks).
await t.db
.update(recordings)
.set({
startedAt: sql`now() - interval '5 minutes'`,
createdAt: sql`now() - interval '1 minute'`,
})
.where(eq(recordings.id, recId));
await setSetting(t.db, 'ai.stt', { url, model: 'whisper' });
await setSetting(t.db, 'ai.llm', { url, model: 'qwen3:4b' });
await setSetting(t.db, 'ai.callTranscripts', true);
await setSetting(t.db, 'ai.callSummaries', true);
const mk = (cookie: string) => (url: string) =>
t.app.inject({ method: 'GET', url, headers: { ...H, cookie } });
as = {
admin: mk(await t.login(ADMIN.email, ADMIN.password)),
alice: mk(await t.login('alice@example.com', 'alice-password-123')),
bob: mk(await t.login('bob@example.com', 'bob-password-123')),
};
});
afterAll(async () => {
await t?.close();
ai?.close();
});
it('without side files (recorded before it was on): marked, nothing asked', async () => {
expect(await transcribeCalls({ db: t.db, dir: dir(), log })).toBe(0);
const [r] = await t.db.select().from(recordings).where(eq(recordings.id, recId));
expect(r).toMatchObject({
aiError: 'Recorded before call transcripts were on',
transcript: null,
});
expect(r!.aiDoneAt).not.toBeNull();
expect(asked).toEqual([]);
});
it('the AI stack down: the call waits for the next run', async () => {
await pending();
writeSides();
down = true;
expect(await transcribeCalls({ db: t.db, dir: dir(), log })).toBe(0);
down = false;
const [r] = await t.db.select().from(recordings).where(eq(recordings.id, recId));
expect(r!.aiDoneAt).toBeNull();
expect(existsSync(join(dir(), sides()[0]!))).toBe(true);
});
it('both sides written out and merged by time, summarized, side files removed', async () => {
expect(await transcribeCalls({ db: t.db, dir: dir(), log })).toBe(1);
const [r] = await t.db.select().from(recordings).where(eq(recordings.id, recId));
expect(r!.transcript).toEqual([
{ side: 'caller', at: 0.4, text: 'Hi, I never got the invoice.' },
{ side: 'other', at: 2.9, text: "I'll email it today." },
{ side: 'caller', at: 6.2, text: 'Great, thanks.' },
]);
expect(r!.summary).toBe(
'The caller never got their invoice; it will be emailed today.\nAction items:\n- Email the invoice today',
);
expect(r!.aiError).toBeNull();
for (const f of sides()) expect(existsSync(join(dir(), f))).toBe(false);
expect(existsSync(join(dir(), FILE))).toBe(true); // the recording itself stays
// done once: a second run asks nothing
const n = asked.length;
await transcribeCalls({ db: t.db, dir: dir(), log });
expect(asked).toHaveLength(n);
});
it('the list shows the summary and finds the call by what was said', async () => {
const list = (await as.admin('/api/recordings')).json();
expect(list.items[0]).toMatchObject({
id: recId,
hasTranscript: true,
summary: expect.stringContaining('invoice'),
});
expect((await as.admin('/api/recordings?q=email%20it')).json().items).toHaveLength(1);
expect((await as.admin('/api/recordings?q=refund')).json().items).toHaveLength(0);
expect((await as.admin('/api/recordings?q=100%25')).json().items).toHaveLength(0); // % literal
});
it('the transcript: admins and the people on the call only', async () => {
const r = await as.alice(`/api/recordings/${recId}/transcript`);
expect(r.statusCode).toBe(200);
expect(r.json().lines).toHaveLength(3);
expect((await as.bob(`/api/recordings/${recId}/transcript`)).statusCode).toBe(403);
expect((await as.admin(`/api/recordings/${recId}/transcript`)).statusCode).toBe(200);
});
it('the dialplan records each side only while call transcripts are on', async () => {
await t.app.generator.apply({ trigger: 'test', force: true });
const globals = () =>
readFileSync(join(t.genDir, 'extensions_globals_selfphoneng.conf'), 'utf8');
expect(globals()).toContain('SP_REC_SIDES=1\n');
await setSetting(t.db, 'ai.callTranscripts', false);
await t.app.generator.apply({ trigger: 'test', force: true });
expect(globals()).toContain('SP_REC_SIDES=\n');
});
});
+7 -2
View File
@@ -87,6 +87,7 @@ import type {
BackupDto,
Warning,
AiTestResult,
TranscriptDto,
} from '@selfphoneng/shared';
export class ApiError extends Error {
@@ -442,8 +443,12 @@ export const reportsApi = {
};
export const recordingsApi = {
list: (cursor?: string) =>
api<ListResponse<RecordingDto>>('GET', `/api/recordings${cursor ? `?cursor=${cursor}` : ''}`),
list: (cursor?: string, q?: string) =>
api<ListResponse<RecordingDto>>(
'GET',
`/api/recordings?${new URLSearchParams({ ...(cursor && { cursor }), ...(q && { q }) })}`,
),
transcript: (id: string) => api<TranscriptDto>('GET', `/api/recordings/${id}/transcript`),
audioUrl: (id: string, download = false) =>
`/api/recordings/${id}/audio${download ? '?download=1' : ''}`,
remove: (id: string) => api<{ ok: true }>('DELETE', `/api/recordings/${id}`),
+87 -7
View File
@@ -1,18 +1,45 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// Recordings (05 §2): every recorded call, newest first; play, download, delete. Calls are
// recorded by user record modes and per phone number; retention is in Settings → Recording.
import { useInfiniteQuery, useMutation, useQueryClient } from '@tanstack/react-query';
// With call transcripts on (spec 10 §3): the summary, the transcript, and search.
import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { useState } from 'react';
import { recordingsApi } from '../../api/client.ts';
import { ConfirmDialog } from '../../components/Dialog.tsx';
import { Alert, Button } from '../../components/ui.tsx';
import { Alert, Button, inputClass } from '../../components/ui.tsx';
import { fmtDuration, fmtNumber } from './CallsPage.tsx';
const mmss = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, '0')}`;
/** Who said what, fetched when opened. */
function Transcript({ id }: { id: string }) {
const t = useQuery({ queryKey: ['transcript', id], queryFn: () => recordingsApi.transcript(id) });
if (t.error) return <Alert>{t.error.message}</Alert>;
if (!t.data) return <p className="text-zinc-500">Loading…</p>;
return (
<ol aria-label="Transcript" className="mt-1 flex flex-col gap-1">
{t.data.lines.map((l, i) => (
<li key={i} className="flex gap-2">
<span className="w-10 shrink-0 text-zinc-500 tabular-nums">{mmss(l.at)}</span>
<span className="w-20 shrink-0 font-medium">
{l.side === 'caller' ? 'Caller' : 'Other side'}
</span>
<span>{l.text}</span>
</li>
))}
{!t.data.lines.length && <li className="text-zinc-500">No words.</li>}
</ol>
);
}
export function RecordingsPage() {
const qc = useQueryClient();
const [draft, setDraft] = useState('');
const [q, setQ] = useState('');
const [open, setOpen] = useState<string | null>(null);
const list = useInfiniteQuery({
queryKey: ['recordings'],
queryFn: ({ pageParam }) => recordingsApi.list(pageParam),
queryKey: ['recordings', q],
queryFn: ({ pageParam }) => recordingsApi.list(pageParam, q || undefined),
initialPageParam: undefined as string | undefined,
getNextPageParam: (last) => last.nextCursor ?? undefined,
});
@@ -30,10 +57,41 @@ export function RecordingsPage() {
<h1 className="text-xl font-semibold">Recordings</h1>
<p className="text-sm text-zinc-500">
Calls are recorded when a user's record mode or a phone number says so. How long they're
kept is under Settings → Recording.
kept is under Settings → Recording. With call transcripts on (Settings → AI), each call is
written out a minute or two after it ends.
</p>
<form
role="search"
className="flex flex-wrap gap-2"
onSubmit={(e) => {
e.preventDefault();
setQ(draft.trim());
}}
>
<input
aria-label="Search transcripts"
placeholder="Search what was said"
className={`${inputClass} max-w-72`}
value={draft}
onChange={(e) => setDraft(e.target.value)}
/>
<Button type="submit">Search</Button>
{q && (
<Button
variant="ghost"
onClick={() => {
setDraft('');
setQ('');
}}
>
Clear
</Button>
)}
</form>
{list.isError && <Alert>{list.error.message}</Alert>}
{list.data && items.length === 0 && <p className="text-sm text-zinc-500">No recordings.</p>}
{list.data && items.length === 0 && (
<p className="text-sm text-zinc-500">{q ? 'No call mentions that.' : 'No recordings.'}</p>
)}
<ul
aria-label="Recordings"
className="divide-y divide-zinc-100 rounded-lg border border-zinc-200 dark:divide-zinc-800 dark:border-zinc-800"
@@ -68,6 +126,28 @@ export function RecordingsPage() {
<Button variant="ghost" onClick={() => setDeleting(r.id)}>
Delete
</Button>
{r.summary && (
<p
aria-label="Summary"
className="w-full whitespace-pre-line text-zinc-700 dark:text-zinc-300"
>
{r.summary}
</p>
)}
{r.aiError && (
<p className="w-full text-xs text-zinc-500">
{r.hasTranscript ? '' : 'No transcript: '}
{r.aiError}
</p>
)}
{r.hasTranscript && (
<div className="w-full">
<Button variant="ghost" onClick={() => setOpen(open === r.id ? null : r.id)}>
{open === r.id ? 'Hide transcript' : 'Show transcript'}
</Button>
{open === r.id && <Transcript id={r.id} />}
</div>
)}
</li>
))}
</ul>
@@ -81,7 +161,7 @@ export function RecordingsPage() {
<ConfirmDialog
open={deleting !== null}
title="Delete recording"
message={<p>Delete this recording? This can't be undone.</p>}
message={<p>Delete this recording and its transcript? This can't be undone.</p>}
confirmLabel="Delete"
busy={remove.isPending}
error={remove.error?.message}
+21
View File
@@ -588,6 +588,8 @@ function AiTab({ s }: { s: SettingsDto }) {
) as Record<'llm' | 'stt' | 'tts', Ep>,
);
const [vmText, setVmText] = useState(s['ai.voicemailTranscripts']);
const [callText, setCallText] = useState(s['ai.callTranscripts']);
const [callSum, setCallSum] = useState(s['ai.callSummaries']);
const [tests, setTests] = useState<Record<string, AiTestResult | 'running'>>({});
const set = (kind: keyof typeof eps, k: keyof Ep, v: string) =>
setEps((e) => ({ ...e, [kind]: { ...e[kind], [k]: v } }));
@@ -615,6 +617,8 @@ function AiTab({ s }: { s: SettingsDto }) {
'ai.stt': value('stt'),
'ai.tts': value('tts'),
'ai.voicemailTranscripts': vmText,
'ai.callTranscripts': callText,
'ai.callSummaries': callText && callSum,
});
}}
>
@@ -684,6 +688,23 @@ function AiTab({ s }: { s: SettingsDto }) {
New messages are written out before the voicemail email goes, and the text is in the email
and on the voicemail page.
</p>
<label className="flex items-center gap-2 text-sm">
<input type="checkbox" checked={callText} onChange={(e) => setCallText(e.target.checked)} />
Call transcripts (speech to text)
</label>
<p className="-mt-2 text-xs text-zinc-500">
Recorded calls are written out a minute or two after they end, the caller and the other side
apart, and can be searched on the Recordings page. Only calls recorded while this is on.
</p>
<label className="flex items-center gap-2 text-sm">
<input
type="checkbox"
checked={callText && callSum}
disabled={!callText}
onChange={(e) => setCallSum(e.target.checked)}
/>
Call summaries and action items (language model)
</label>
<div className="flex items-center gap-3">
<Button type="submit" variant="primary" disabled={save.isPending}>
Save