ai: Settings → AI (local endpoints only, with tests), the AI stack, voicemail transcripts

apps/api/src/ai/client.test.ts

@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { createServer, type IncomingMessage, type Server } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { assertLocalUrl, chat, silentWav, speak, testEndpoint, transcribe } from './client.ts';
+
+describe('assertLocalUrl', () => {
+  it.each([
+    'http://127.0.0.1:11434/v1',
+    'http://localhost:8000/v1',
+    'https://192.168.1.50/v1',
+    'http://10.0.0.5:8000',
+    'http://172.20.0.1:8000',
+    'http://[::1]:8000/v1',
+    'http://[fd00::5]/v1',
+  ])('%s is local', (u) => expect(() => assertLocalUrl(u)).not.toThrow());
+
+  it.each([
+    ['http://api.openai.com/v1', /use an IP address/],
+    ['http://ollama.lan:11434', /use an IP address/],
+    ['http://8.8.8.8/v1', /not on this server or the local network/],
+    ['http://100.64.1.2/v1', /not on this server/], // CGNAT / Tailscale: may cross the internet
+    ['http://172.32.0.1/v1', /not on this server/],
+    ['ftp://127.0.0.1/', /only http/],
+    ['nonsense', /not an address/],
+  ])('%s is refused', (u, msg) => expect(() => assertLocalUrl(u)).toThrow(msg));
+});
+
+describe('OpenAI-style calls', () => {
+  let server: Server;
+  let base: string;
+  const seen: { path: string; type: string; body: string }[] = [];
+  const read = (req: IncomingMessage) =>
+    new Promise<string>((r) => {
+      let b = '';
+      req.setEncoding('latin1').on('data', (c: string) => (b += c));
+      req.on('end', () => r(b));
+    });
+
+  beforeAll(async () => {
+    server = createServer((req, res) => {
+      void read(req).then((body) => {
+        seen.push({ path: req.url!, type: req.headers['content-type'] ?? '', body });
+        if (req.url === '/v1/audio/transcriptions') return res.end('{"text":"  Call me back. "}');
+        if (req.url === '/v1/audio/speech') return res.end(silentWav());
+        if (req.url === '/v1/chat/completions')
+          return res.end(
+            JSON.stringify({
+              choices: [{ message: { content: '<think>\nhm\n</think>\n\n OK ' } }],
+            }),
+          );
+        res.writeHead(404).end('model "nope" not found');
+      });
+    });
+    await new Promise<void>((r) => server.listen(0, '127.0.0.1', r));
+    base = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1`;
+  });
+  afterAll(() => server.close());
+
+  it('transcribe: multipart with the file and model, the text trimmed', async () => {
+    expect(await transcribe({ url: base, model: 'whisper' }, silentWav())).toBe('Call me back.');
+    const r = seen.at(-1)!;
+    expect(r.type).toMatch(/^multipart\/form-data; boundary=/);
+    expect(r.body).toContain('name="model"\r\n\r\nwhisper');
+    expect(r.body).toContain('filename="audio.wav"');
+    expect(r.body).toContain('RIFF');
+  });
+
+  it('speak: JSON in, WAV out; chat: the answer', async () => {
+    const wav = await speak({ url: `${base}/`, model: 'kokoro', voice: 'af_heart' }, 'Hi');
+    expect(wav.subarray(0, 4).toString()).toBe('RIFF');
+    expect(JSON.parse(seen.at(-1)!.body)).toEqual({
+      model: 'kokoro',
+      voice: 'af_heart',
+      input: 'Hi',
+      response_format: 'wav',
+    });
+    expect(await chat({ url: base, model: 'm' }, [{ role: 'user', content: 'x' }])).toBe('OK');
+  });
+
+  it('errors say what went wrong; tests report them', async () => {
+    await expect(transcribe({ url: `${base}/wrong`, model: 'm' }, silentWav())).rejects.toThrow(
+      /wrong\/audio\/transcriptions: HTTP 404: model "nope" not found/,
+    );
+    await expect(
+      transcribe({ url: 'http://127.0.0.1:1/v1', model: 'm' }, silentWav()),
+    ).rejects.toThrow(/Can't reach 127.0.0.1:1/);
+    expect(await testEndpoint('stt', { url: 'http://8.8.8.8/v1', model: 'm' })).toEqual({
+      ok: false,
+      error: '8.8.8.8 is not on this server or the local network',
+    });
+    const tts = await testEndpoint('tts', { url: base, model: 'k', voice: 'v' });
+    expect(tts).toMatchObject({
+      ok: true,
+      audio: expect.stringMatching(/^data:audio\/wav;base64,UklG/),
+    });
+  });
+});

apps/api/src/ai/client.ts

@@ -0,0 +1,174 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Local AI (spec 10): OpenAI-style APIs (Ollama, Speaches, llama.cpp, …) on this server or the
+// local network. Nothing else is called: an address must be an IP on a local network or
+// localhost, written as such, so no DNS answer can send audio or text anywhere else.
+import { BlockList, isIP } from 'node:net';
+import type { AiEndpoint, AiTestResult, AiTtsEndpoint } from '@selfphoneng/shared';
+
+export class AiError extends Error {}
+
+const LOCAL = new BlockList();
+for (const [net, bits] of [
+  ['127.0.0.0', 8],
+  ['10.0.0.0', 8],
+  ['172.16.0.0', 12],
+  ['192.168.0.0', 16],
+] as const)
+  LOCAL.addSubnet(net, bits, 'ipv4');
+for (const [net, bits] of [
+  ['::1', 128],
+  ['fc00::', 7],
+  ['fe80::', 10],
+] as const)
+  LOCAL.addSubnet(net, bits, 'ipv6');
+
+/** Throws unless `url` is http(s) to localhost or a local-network IP. */
+export function assertLocalUrl(url: string): void {
+  let u: URL;
+  try {
+    u = new URL(url);
+  } catch {
+    throw new AiError(`${url} is not an address`);
+  }
+  const host = u.hostname.replace(/^\[|\]$/g, '');
+  if (!/^https?:$/.test(u.protocol)) throw new AiError(`${url}: only http and https`);
+  if (host === 'localhost') return;
+  const v = isIP(host);
+  if (!v)
+    throw new AiError(
+      `${host}: use an IP address (or localhost), so the address can't lead off this network`,
+    );
+  if (!LOCAL.check(host, v === 6 ? 'ipv6' : 'ipv4'))
+    throw new AiError(`${host} is not on this server or the local network`);
+}
+
+async function call(
+  base: string,
+  path: string,
+  init: RequestInit,
+  timeoutMs: number,
+): Promise<Response> {
+  assertLocalUrl(base);
+  const url = new URL(path, base.endsWith('/') ? base : `${base}/`);
+  let r: Response;
+  try {
+    r = await fetch(url, { ...init, redirect: 'error', signal: AbortSignal.timeout(timeoutMs) });
+  } catch (err) {
+    const e = err as Error & { cause?: { code?: string } };
+    throw new AiError(
+      e.name === 'TimeoutError'
+        ? `${url.host} didn't answer within ${timeoutMs / 1000} s`
+        : `Can't reach ${url.host}${e.cause?.code ? ` (${e.cause.code})` : ''}`,
+    );
+  }
+  if (!r.ok) {
+    const body = (await r.text().catch(() => '')).slice(0, 300);
+    throw new AiError(`${url.pathname}: HTTP ${r.status}${body ? `: ${body}` : ''}`);
+  }
+  return r;
+}
+
+/** Speech to text: the words in a WAV (or any format the server reads). */
+export async function transcribe(ep: AiEndpoint, audio: Buffer, timeoutMs = 180_000) {
+  const form = new FormData();
+  form.append('file', new Blob([new Uint8Array(audio)], { type: 'audio/wav' }), 'audio.wav');
+  form.append('model', ep.model);
+  form.append('response_format', 'json');
+  const r = await call(ep.url, 'audio/transcriptions', { method: 'POST', body: form }, timeoutMs);
+  const { text } = (await r.json()) as { text?: unknown };
+  if (typeof text !== 'string') throw new AiError('audio/transcriptions: no text in the answer');
+  return text.trim();
+}
+
+/** Text to speech: a WAV. */
+export async function speak(ep: AiTtsEndpoint, input: string, timeoutMs = 60_000) {
+  const r = await call(
+    ep.url,
+    'audio/speech',
+    {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ model: ep.model, voice: ep.voice, input, response_format: 'wav' }),
+    },
+    timeoutMs,
+  );
+  return Buffer.from(await r.arrayBuffer());
+}
+
+/** One chat answer from the LLM. */
+export async function chat(
+  ep: AiEndpoint,
+  messages: { role: 'system' | 'user' | 'assistant'; content: string }[],
+  opts: { maxTokens?: number; timeoutMs?: number } = {},
+) {
+  const r = await call(
+    ep.url,
+    'chat/completions',
+    {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        model: ep.model,
+        messages,
+        max_tokens: opts.maxTokens,
+        stream: false,
+      }),
+    },
+    opts.timeoutMs ?? 120_000,
+  );
+  const j = (await r.json()) as { choices?: { message?: { content?: unknown } }[] };
+  const content = j.choices?.[0]?.message?.content;
+  if (typeof content !== 'string') throw new AiError('chat/completions: no answer');
+  // Thinking models (Qwen3, …) may put their reasoning inline; only the answer is wanted.
+  return content.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+}
+
+/** One second of 8 kHz silence: enough for a speech-to-text server to load its model and answer. */
+export function silentWav(seconds = 1): Buffer {
+  const n = 8000 * seconds;
+  const b = Buffer.alloc(44 + n * 2);
+  b.write('RIFF', 0);
+  b.writeUInt32LE(36 + n * 2, 4);
+  b.write('WAVEfmt ', 8);
+  b.writeUInt32LE(16, 16);
+  b.writeUInt16LE(1, 20); // PCM
+  b.writeUInt16LE(1, 22); // mono
+  b.writeUInt32LE(8000, 24);
+  b.writeUInt32LE(16000, 28);
+  b.writeUInt16LE(2, 32);
+  b.writeUInt16LE(16, 34);
+  b.write('data', 36);
+  b.writeUInt32LE(n * 2, 40);
+  return b;
+}
+
+/** Settings → AI → Test: a tiny real request. The first one may load a model, so it can be slow. */
+export async function testEndpoint(
+  kind: 'llm' | 'stt' | 'tts',
+  ep: AiEndpoint | AiTtsEndpoint,
+): Promise<AiTestResult> {
+  const t0 = Date.now();
+  try {
+    if (kind === 'llm') {
+      const a = await chat(ep, [{ role: 'user', content: 'Reply with just: OK' }], {
+        maxTokens: 400, // room for a thinking model to think first
+      });
+      return { ok: true, ms: Date.now() - t0, detail: `Answered: ${a.slice(0, 80)}` };
+    }
+    if (kind === 'stt') {
+      await transcribe(ep, silentWav());
+      return { ok: true, ms: Date.now() - t0, detail: 'Transcribed a second of silence' };
+    }
+    if (!('voice' in ep)) throw new AiError('A voice is needed');
+    const wav = await speak(ep, 'Hello, this is your phone system.');
+    if (wav.length < 1000) throw new AiError('audio/speech: the audio is empty');
+    return {
+      ok: true,
+      ms: Date.now() - t0,
+      detail: `${Math.round(wav.length / 1024)} KB of audio`,
+      audio: `data:audio/wav;base64,${wav.toString('base64')}`,
+    };
+  } catch (err) {
+    return { ok: false, error: err instanceof AiError ? err.message : String(err) };
+  }
+}

apps/api/src/db/migrations/0006_voicemail_transcript.sql

@@ -0,0 +1 @@
+ALTER TABLE "voicemail_messages" ADD COLUMN "transcript" text;

apps/api/src/db/migrations/meta/_journal.json

@@ -43,6 +43,13 @@
       "when": 1792606712441,
       "tag": "0005_moh_order",
       "breakpoints": true
+    },
+    {
+      "idx": 6,
+      "version": "7",
+      "when": 1792624567407,
+      "tag": "0006_voicemail_transcript",
+      "breakpoints": true
     }
   ]
 }

apps/api/src/db/schema.ts

@@ -590,6 +590,8 @@ export const voicemailMessages = pgTable('voicemail_messages', {
   filePath: text().notNull(),
   emailedAt: tstz(),
   deletedAt: tstz(),
+  /** Speech to text (spec 10), when ai.voicemailTranscripts is on; null: none (yet). */
+  transcript: text(),
   ...timestamps,
 });
 

apps/api/src/routes/settings.ts

@@ -4,8 +4,11 @@ import { eq, inArray } from 'drizzle-orm';
 import type { FastifyInstance } from 'fastify';
 import { z } from 'zod';
 import {
+  AiEndpointSchema,
+  AiTtsEndpointSchema,
   SettingsSchemas,
   SmtpInputSchema,
+  type AiTestResult,
   type Destination,
   type SettingKey,
   type SettingsDto,
@@ -24,6 +27,7 @@ import { readTlsCert } from '../services/tls.ts';
 import { backupMissed } from '../services/backup.ts';
 import { detectPublicIp } from '../jobs.ts';
 import type { SendMail } from '../services/mail.ts';
+import { AiError, assertLocalUrl, testEndpoint } from '../ai/client.ts';
 
 declare module 'fastify' {
   interface FastifyInstance {
@@ -114,6 +118,15 @@ export async function settingsRoutes(app: FastifyInstance) {
       } else values[key] = SettingsSchemas[key as SettingKey].parse(raw);
     }
     await checkRefs(db, values);
+    for (const key of ['ai.llm', 'ai.stt', 'ai.tts'] as const) {
+      const ep = values[key] as { url: string } | null | undefined;
+      try {
+        if (ep) assertLocalUrl(ep.url);
+      } catch (err) {
+        if (err instanceof AiError) throw badRequest(`${key}: ${err.message}`);
+        throw err;
+      }
+    }
     const before = await getAllSettings(db);
     // SIP ports may not land on another listener (Asterisk would fail to start the transport).
     const c = app.config;
@@ -214,6 +227,15 @@ export async function settingsRoutes(app: FastifyInstance) {
     return { ok: true };
   });
 
+  /** Settings → AI → Test (spec 10): the values in the form, saved or not. */
+  app.post('/api/ai/test', async (req): Promise<AiTestResult> => {
+    const { kind } = z.object({ kind: z.enum(['llm', 'stt', 'tts']) }).parse(req.body);
+    const ep = (kind === 'tts' ? AiTtsEndpointSchema : AiEndpointSchema).parse(
+      (req.body as { endpoint?: unknown }).endpoint,
+    );
+    return testEndpoint(kind, ep);
+  });
+
   /** The SIP TLS certificate, for Settings → Security (M11-T2). */
   app.get('/api/system/tls', () => readTlsCert(app.config.TLS_DIR));
 

apps/api/src/services/settings.ts

@@ -6,8 +6,20 @@ import type { Db } from '../db/client.ts';
 import { settings } from '../db/schema.ts';
 import { randomAlnum } from '../crypto/secrets.ts';
 
+/** AI_<KIND>_URL and _MODEL (and AI_TTS_VOICE) start an install with its AI endpoints set. */
+function aiFromEnv(kind: 'LLM' | 'STT' | 'TTS') {
+  const e = process.env;
+  const [url, model, voice] = [e[`AI_${kind}_URL`], e[`AI_${kind}_MODEL`], e.AI_TTS_VOICE];
+  if (!url || !model || (kind === 'TTS' && !voice)) return null;
+  return kind === 'TTS' ? { url, model, voice } : { url, model };
+}
+
 function defaultFor<K extends SettingKey>(key: K): SettingValue<K> {
   if (key === 'general.timezone' && process.env.TZ) return process.env.TZ as SettingValue<K>;
+  if (key === 'ai.llm' || key === 'ai.stt' || key === 'ai.tts') {
+    const v = aiFromEnv(key === 'ai.llm' ? 'LLM' : key === 'ai.stt' ? 'STT' : 'TTS');
+    if (v) return SettingsSchemas[key].parse(v) as SettingValue<K>;
+  }
   return SettingsSchemas[key].parse(undefined) as SettingValue<K>;
 }
 

apps/api/src/services/voicemail.ts

@@ -17,6 +17,7 @@ import {
 import { field, type AmiMessage } from '../ami/events.ts';
 import type { Db } from '../db/client.ts';
 import { extensions, users, voicemailMessages } from '../db/schema.ts';
+import { transcribe } from '../ai/client.ts';
 import { MailNotConfigured, type SendMail } from './mail.ts';
 import { getSetting } from './settings.ts';
 import { enqueueWebhook } from './webhooks.ts';
@@ -60,6 +61,7 @@ export const toVoicemailDto = (r: Row): VoicemailDto => ({
   callerName: r.callerName,
   durationS: r.durationS,
   receivedAt: r.receivedAt.toISOString(),
+  transcript: r.transcript,
 });
 
 /** Everything in one mailbox's INBOX, Old and Urgent folders. filePath is relative to the spool. */
@@ -99,7 +101,26 @@ type Box = {
   voicemailDeleteAfterEmail: boolean;
 };
 
-async function announce(d: VoicemailDeps, box: Box, row: Row) {
+/** Speech to text before anyone is told, so the email has the words. Null: off, or it failed. */
+async function transcribeMessage(d: VoicemailDeps, row: Row): Promise<string | null> {
+  const stt = await getSetting(d.db, 'ai.stt');
+  if (!stt || !(await getSetting(d.db, 'ai.voicemailTranscripts'))) return null;
+  try {
+    const text = await transcribe(stt, await readFile(join(d.spoolDir, row.filePath)));
+    await d.db
+      .update(voicemailMessages)
+      .set({ transcript: text })
+      .where(eq(voicemailMessages.id, row.id));
+    return text;
+  } catch (err) {
+    d.log.error({ err, msgId: row.msgId }, 'voicemail transcript failed');
+    return null;
+  }
+}
+
+async function announce(d: VoicemailDeps, box: Box, found: Row) {
+  const transcript = await transcribeMessage(d, found);
+  const row = { ...found, transcript };
   // Every mailbox, with or without a user (the live message is only for the user's own).
   void enqueueWebhook(d.db, 'voicemail.new', {
     ...toVoicemailDto(row),
@@ -125,6 +146,9 @@ async function announce(d: VoicemailDeps, box: Box, row: Row) {
         `Number: ${row.callerNumber ? formatNumber(row.callerNumber) : 'unknown'}`,
         `Received: ${at}`,
         `Length: ${mmss(row.durationS)}`,
+        ...(transcript === null
+          ? []
+          : ['', 'What they said (automatic, may have mistakes):', transcript || '(no words)']),
         '',
         `Listen: https://${d.publicHost}/me/voicemail`,
       ].join('\n'),

apps/api/test/voicemail.test.ts

@@ -1,12 +1,15 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // M4-T2: voicemail index, live event and email (04 §5.6, §10).
 import { mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { createServer } from 'node:http';
+import type { AddressInfo } from 'node:net';
 import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
 import { afterAll, beforeAll, describe, expect, it } from 'vitest';
 import type { LiveMessage } from '@selfphoneng/shared';
 import { extensions, voicemailMessages } from '../src/db/schema.ts';
 import { parseCallerId, type watchVoicemail } from '../src/services/voicemail.ts';
+import { setSetting } from '../src/services/settings.ts';
 import { hasTestDb } from './db.ts';
 import { ADMIN, H, createTestApp } from './app.ts';
 
@@ -159,4 +162,45 @@ describe.skipIf(!hasTestDb)('voicemail watcher', () => {
     const r = (await rows()).find((x) => x.msgId === '1792300600-00000003')!;
     expect(r).toMatchObject({ emailedAt: null, deletedAt: null });
   });
+
+  it('spec 10: with transcripts on, the words go in the row, the email and the list', async () => {
+    let fail = false;
+    const stt = createServer((req, res) => {
+      req.resume();
+      req.on('end', () =>
+        fail ? res.writeHead(500).end('out of memory') : res.end('{"text":" Please call me. "}'),
+      );
+    });
+    await new Promise<void>((ok) => stt.listen(0, '127.0.0.1', ok));
+    const url = `http://127.0.0.1:${(stt.address() as AddressInfo).port}/v1`;
+    await setSetting(t.db, 'ai.stt', { url, model: 'whisper' });
+    await setSetting(t.db, 'ai.voicemailTranscripts', true);
+    try {
+      leave('INBOX', 3, '1792300700-00000004', '<104>');
+      await w.scan('101');
+      const r = (await rows()).find((x) => x.msgId === '1792300700-00000004')!;
+      expect(r.transcript).toBe('Please call me.');
+      expect(t.mails.at(-1)!.text).toContain(
+        'What they said (automatic, may have mistakes):\nPlease call me.',
+      );
+      const list = await t.app.inject({
+        method: 'GET',
+        url: '/api/me/voicemail',
+        headers: { ...H, cookie: await t.login('dan@example.com', 'dan-password-123') },
+      });
+      expect(list.json().items).toContainEqual(
+        expect.objectContaining({ id: r.id, transcript: 'Please call me.' }),
+      );
+
+      // The speech server failing doesn't hold up the email.
+      fail = true;
+      leave('INBOX', 4, '1792300800-00000005', '<105>');
+      await w.scan('101');
+      expect((await rows()).find((x) => x.msgId === '1792300800-00000005')!.transcript).toBeNull();
+      expect(t.mails.at(-1)!.text).not.toContain('What they said');
+    } finally {
+      await setSetting(t.db, 'ai.voicemailTranscripts', false);
+      stt.close();
+    }
+  });
 });