ai: call transcripts (each side recorded, merged by time) and summaries; search on Recordings

apps/api/test/asterisk/rec-test.ts

@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Call transcripts (spec 10 §3): with ai.callTranscripts on, sp-sub-record writes the mixed
+// recording plus <call>-in.wav (what the caller said) and <call>-out.wav (what they heard). A Local
+// channel records itself through sp-sub-record and dials one that answers and stays silent; the
+// other half plays digits as "the caller". So -in must have sound and -out (nearly) none.
+// Usage: node --conditions=source apps/api/test/asterisk/rec-test.ts   (ASTERISK_IMAGE)
+import { spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { SettingsSchemas, type SettingKey } from '@selfphoneng/shared';
+import { GENERATED_FILES, render } from '../../src/generator/index.ts';
+import type { Snapshot } from '../../src/generator/load.ts';
+
+const IMAGE = process.env.ASTERISK_IMAGE ?? 'selfphoneng-asterisk';
+const AST = 'sp-test-rec';
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+const docker = (...args: string[]) => {
+  const r = spawnSync('docker', args, { encoding: 'utf8' });
+  return { ok: r.status === 0, out: `${r.stdout ?? ''}${r.stderr ?? ''}` };
+};
+const cli = (cmd: string) => docker('exec', AST, 'asterisk', '-rx', cmd).out;
+const log = (...a: unknown[]) => console.log('[rec-test]', ...a);
+
+function snapshot(): Snapshot {
+  const settings = Object.fromEntries(
+    (Object.keys(SettingsSchemas) as SettingKey[])
+      .filter((k) => k !== 'sms.webhookToken')
+      .map((k) => [k, SettingsSchemas[k].parse(undefined)]),
+  ) as Snapshot['settings'];
+  settings['ai.callTranscripts'] = true;
+  settings['ai.stt'] = { url: 'http://127.0.0.1:8000/v1', model: 'whisper' };
+  return {
+    today: '2026-10-21',
+    settings,
+    dids: [],
+    trunks: [],
+    routes: [],
+    ringGroups: [],
+    queues: [],
+    conferences: [],
+    pagingGroups: [],
+    ivrs: [],
+    timeConditions: [],
+    holidays: [],
+    announcements: [],
+    recordingAnnouncement: null,
+    extensions: [],
+    mohClasses: [],
+  };
+}
+
+async function main() {
+  const dir = mkdtempSync(join(tmpdir(), 'sp-rec-'));
+  const conf = join(dir, 'conf');
+  mkdirSync(conf);
+  const files = render(snapshot());
+  for (const f of GENERATED_FILES) writeFileSync(join(conf, f), files[f]);
+  const problems: string[] = [];
+  try {
+    docker('rm', '-f', AST);
+    const run = docker(
+      'run',
+      '-d',
+      '--rm',
+      '--name',
+      AST,
+      '-e',
+      'AMI_SECRET=rec-test-secret',
+      '-v',
+      `${conf}:/etc/asterisk/selfphoneng`,
+      IMAGE,
+    );
+    if (!run.ok) throw new Error(`asterisk: ${run.out}`);
+    for (
+      let i = 0;
+      i < 60 && !docker('exec', AST, 'asterisk', '-rx', 'core waitfullybooted').ok;
+      i++
+    )
+      await sleep(500);
+    log(
+      cli('dialplan show sp-sub-record')
+        .split('\n')
+        .filter((l) => l.includes('MixMonitor'))
+        .join(' | '),
+    );
+    for (const line of [
+      'dialplan add extension rec,1,Gosub,"sp-sub-record,s,1" into sp-rectest',
+      'dialplan add extension rec,2,Dial,"Local/play@sp-rectest/n" into sp-rectest',
+      'dialplan add extension play,1,Answer, into sp-rectest',
+      'dialplan add extension play,2,Wait,4 into sp-rectest',
+    ])
+      cli(line);
+    cli('channel originate Local/rec@sp-rectest/n application Playback digits/1&digits/2&digits/3');
+    await sleep(6000);
+    const ls = docker(
+      'exec',
+      AST,
+      'sh',
+      '-c',
+      'find /var/spool/asterisk/monitor -name "*.wav" -exec stat -c "%s %n" {} \\;',
+    ).out.trim();
+    log(ls.replaceAll('\n', ' | ') || 'no files');
+    const sized = ls
+      .split('\n')
+      .filter(Boolean)
+      .map((l) => {
+        const [size, name] = l.split(' ');
+        return { size: Number(size), name: name! };
+      });
+    const mixed = sized.find((f) => /\/\d+\.\d+\.wav$/.test(f.name));
+    if (!mixed) problems.push('no mixed recording');
+    else {
+      for (const side of ['-in.wav', '-out.wav']) {
+        const f = sized.find((x) => x.name === mixed.name.replace(/\.wav$/, side));
+        if (!f) problems.push(`no ${side} next to ${mixed.name}`);
+        else if (f.size < 8000) problems.push(`${side} has no audio (${f.size} bytes)`);
+      }
+      // Each side on its own, the right way round: the caller's file has the digits, the silent
+      // side's has nothing (a copy of the mix would have them too).
+      const loud = (side: string) => {
+        const wav = spawnSync('docker', ['exec', AST, 'cat', mixed.name.replace(/\.wav$/, side)])
+          .stdout as Buffer;
+        let sum = 0;
+        for (let i = 44; i + 1 < wav.length; i += 2) sum += Math.abs(wav.readInt16LE(i));
+        return Math.round(sum / ((wav.length - 44) / 2));
+      };
+      const [mix, caller, other] = [loud('.wav'), loud('-in.wav'), loud('-out.wav')];
+      log(`average level: mix ${mix}, -in ${caller}, -out ${other}`);
+      if (!(mix > 200 && caller > 200 && other < caller / 10))
+        problems.push(
+          `-in should be the caller (loud), -out the silent side: ${caller} / ${other}`,
+        );
+    }
+  } catch (e) {
+    problems.push((e as Error).message);
+  } finally {
+    docker('rm', '-f', AST);
+    rmSync(dir, { recursive: true, force: true });
+  }
+  if (problems.length) {
+    console.log('FAIL call recording sides');
+    for (const p of problems) console.log(`     ${p}`);
+    process.exit(1);
+  }
+  console.log('ok   call recording sides (mixed, -in and -out with audio)');
+}
+
+await main();

apps/api/test/audio.ts

@@ -22,3 +22,12 @@ export function silenceWav(seconds = 1) {
   h.writeUInt32LE(data.length, 40);
   return Buffer.concat([h, data]);
 }
+
+/** `silence` s of silence, then `seconds` s of a 440 Hz tone, as an 8 kHz mono 16-bit wav. */
+export function toneWav(seconds = 1, silence = 0) {
+  const w = silenceWav(seconds + silence);
+  const start = 44 + Math.round(silence * 8000) * 2;
+  for (let i = 0; start + i * 2 < w.length; i++)
+    w.writeInt16LE(Math.round(6000 * Math.sin((2 * Math.PI * 440 * i) / 8000)), start + i * 2);
+  return w;
+}

apps/api/test/call-transcripts.test.ts

@@ -0,0 +1,203 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Spec 10 §3: call transcripts and summaries. The job writes out each side of a recorded call,
+// merges them by time, asks the language model for a summary and removes the side files; the
+// Recordings page can search them, and only the people on the call (and admins) can read them.
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { createServer, type Server } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { join } from 'node:path';
+import { eq, sql } from 'drizzle-orm';
+import type { LightMyRequestResponse } from 'fastify';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import type { AmiMessage } from '../src/ami/events.ts';
+import { transcribeCalls } from '../src/ai/calls.ts';
+import { encrypt } from '../src/crypto/secrets.ts';
+import { cdr, extensions, recordings } from '../src/db/schema.ts';
+import { aggregateCall, toCdrRow } from '../src/services/cdr.ts';
+import { setSetting } from '../src/services/settings.ts';
+import { silenceWav, toneWav } from './audio.ts';
+import { hasTestDb } from './db.ts';
+import { ADMIN, H, createTestApp } from './app.ts';
+
+const FILE = '2026/10/21/1792351748.9.wav';
+
+describe.skipIf(!hasTestDb)('call transcripts and summaries', () => {
+  let t: Awaited<ReturnType<typeof createTestApp>>;
+  let ai: Server;
+  let down = false;
+  const asked: string[] = [];
+  let as: Record<'admin' | 'alice' | 'bob', (url: string) => Promise<LightMyRequestResponse>>;
+  let recId: string;
+  const dir = () => t.app.config.RECORDINGS_DIR;
+  const log = { error: () => {} };
+  const sides = () => [`${FILE.slice(0, -4)}-in.wav`, `${FILE.slice(0, -4)}-out.wav`];
+  /** The side files: someone speaking (a tone) on each; the other side speaks longer. */
+  const writeSides = () => {
+    writeFileSync(join(dir(), sides()[0]!), toneWav(1));
+    writeFileSync(join(dir(), sides()[1]!), toneWav(2));
+  };
+  const pending = () =>
+    t.db
+      .update(recordings)
+      .set({ aiDoneAt: null, aiError: null, transcript: null, summary: null })
+      .where(eq(recordings.id, recId));
+
+  beforeAll(async () => {
+    ai = createServer((req, res) => {
+      let body = '';
+      req.setEncoding('latin1').on('data', (c: string) => (body += c));
+      req.on('end', () => {
+        if (down) return res.destroy();
+        asked.push(req.url!);
+        if (req.url === '/v1/audio/transcriptions')
+          return res.end(
+            JSON.stringify(
+              body.length < 30_000 // the caller's stretch: about a second
+                ? {
+                    text: '…',
+                    segments: [
+                      { start: 0.4, text: ' Hi, I never got the invoice.' },
+                      { start: 6.2, text: ' Great, thanks.' },
+                    ],
+                  }
+                : { text: '…', segments: [{ start: 2.9, text: " I'll email it today." }] },
+            ),
+          );
+        if (req.url === '/v1/chat/completions') {
+          expect(body).toContain('Caller: Hi, I never got the invoice.\\nOther side: I');
+          return res.end(
+            JSON.stringify({
+              choices: [
+                {
+                  message: {
+                    content:
+                      '<think>short call</think>The caller never got their invoice; it will be emailed today.\nAction items:\n- Email the invoice today',
+                  },
+                },
+              ],
+            }),
+          );
+        }
+        res.writeHead(404).end();
+      });
+    });
+    await new Promise<void>((r) => ai.listen(0, '127.0.0.1', r));
+    const url = `http://127.0.0.1:${(ai.address() as AddressInfo).port}/v1`;
+
+    t = await createTestApp();
+    await t.clearMustChange(ADMIN.email);
+    const alice = await t.addUser('alice@example.com', 'alice-password-123');
+    await t.addUser('bob@example.com', 'bob-password-123');
+    await t.db.insert(extensions).values({
+      number: '101',
+      name: 'Alice',
+      userId: alice.id,
+      sipPasswordEnc: encrypt('x'),
+      voicemailPinEnc: encrypt('4826'),
+    });
+    const events: AmiMessage[] = JSON.parse(
+      readFileSync(new URL('./fixtures/cdr/inbound-user.json', import.meta.url), 'utf8'),
+    );
+    for (const e of events) await t.db.insert(cdr).values(toCdrRow({ ...e, sp_recording: FILE }));
+    mkdirSync(join(dir(), '2026/10/21'), { recursive: true });
+    writeFileSync(join(dir(), FILE), silenceWav(2));
+    await aggregateCall(t.db, String(events[0]!.LinkedID), { recordingsDir: dir() });
+    const [r] = await t.db.select().from(recordings);
+    recId = r!.id;
+    // A call that ended a minute ago (the fixture's date is older than the job looks).
+    await t.db
+      .update(recordings)
+      .set({
+        startedAt: sql`now() - interval '5 minutes'`,
+        createdAt: sql`now() - interval '1 minute'`,
+      })
+      .where(eq(recordings.id, recId));
+    await setSetting(t.db, 'ai.stt', { url, model: 'whisper' });
+    await setSetting(t.db, 'ai.llm', { url, model: 'qwen3:4b' });
+    await setSetting(t.db, 'ai.callTranscripts', true);
+    await setSetting(t.db, 'ai.callSummaries', true);
+    const mk = (cookie: string) => (url: string) =>
+      t.app.inject({ method: 'GET', url, headers: { ...H, cookie } });
+    as = {
+      admin: mk(await t.login(ADMIN.email, ADMIN.password)),
+      alice: mk(await t.login('alice@example.com', 'alice-password-123')),
+      bob: mk(await t.login('bob@example.com', 'bob-password-123')),
+    };
+  });
+  afterAll(async () => {
+    await t?.close();
+    ai?.close();
+  });
+
+  it('without side files (recorded before it was on): marked, nothing asked', async () => {
+    expect(await transcribeCalls({ db: t.db, dir: dir(), log })).toBe(0);
+    const [r] = await t.db.select().from(recordings).where(eq(recordings.id, recId));
+    expect(r).toMatchObject({
+      aiError: 'Recorded before call transcripts were on',
+      transcript: null,
+    });
+    expect(r!.aiDoneAt).not.toBeNull();
+    expect(asked).toEqual([]);
+  });
+
+  it('the AI stack down: the call waits for the next run', async () => {
+    await pending();
+    writeSides();
+    down = true;
+    expect(await transcribeCalls({ db: t.db, dir: dir(), log })).toBe(0);
+    down = false;
+    const [r] = await t.db.select().from(recordings).where(eq(recordings.id, recId));
+    expect(r!.aiDoneAt).toBeNull();
+    expect(existsSync(join(dir(), sides()[0]!))).toBe(true);
+  });
+
+  it('both sides written out and merged by time, summarized, side files removed', async () => {
+    expect(await transcribeCalls({ db: t.db, dir: dir(), log })).toBe(1);
+    const [r] = await t.db.select().from(recordings).where(eq(recordings.id, recId));
+    expect(r!.transcript).toEqual([
+      { side: 'caller', at: 0.4, text: 'Hi, I never got the invoice.' },
+      { side: 'other', at: 2.9, text: "I'll email it today." },
+      { side: 'caller', at: 6.2, text: 'Great, thanks.' },
+    ]);
+    expect(r!.summary).toBe(
+      'The caller never got their invoice; it will be emailed today.\nAction items:\n- Email the invoice today',
+    );
+    expect(r!.aiError).toBeNull();
+    for (const f of sides()) expect(existsSync(join(dir(), f))).toBe(false);
+    expect(existsSync(join(dir(), FILE))).toBe(true); // the recording itself stays
+    // done once: a second run asks nothing
+    const n = asked.length;
+    await transcribeCalls({ db: t.db, dir: dir(), log });
+    expect(asked).toHaveLength(n);
+  });
+
+  it('the list shows the summary and finds the call by what was said', async () => {
+    const list = (await as.admin('/api/recordings')).json();
+    expect(list.items[0]).toMatchObject({
+      id: recId,
+      hasTranscript: true,
+      summary: expect.stringContaining('invoice'),
+    });
+    expect((await as.admin('/api/recordings?q=email%20it')).json().items).toHaveLength(1);
+    expect((await as.admin('/api/recordings?q=refund')).json().items).toHaveLength(0);
+    expect((await as.admin('/api/recordings?q=100%25')).json().items).toHaveLength(0); // % literal
+  });
+
+  it('the transcript: admins and the people on the call only', async () => {
+    const r = await as.alice(`/api/recordings/${recId}/transcript`);
+    expect(r.statusCode).toBe(200);
+    expect(r.json().lines).toHaveLength(3);
+    expect((await as.bob(`/api/recordings/${recId}/transcript`)).statusCode).toBe(403);
+    expect((await as.admin(`/api/recordings/${recId}/transcript`)).statusCode).toBe(200);
+  });
+
+  it('the dialplan records each side only while call transcripts are on', async () => {
+    await t.app.generator.apply({ trigger: 'test', force: true });
+    const globals = () =>
+      readFileSync(join(t.genDir, 'extensions_globals_selfphoneng.conf'), 'utf8');
+    expect(globals()).toContain('SP_REC_SIDES=1\n');
+    await setSetting(t.db, 'ai.callTranscripts', false);
+    await t.app.generator.apply({ trigger: 'test', force: true });
+    expect(globals()).toContain('SP_REC_SIDES=\n');
+  });
+});