jobs: public IP, trunk and disk watches, retention

apps/api/src/jobs.ts takes over the once-a-minute timer from index.ts
(no croner: every schedule in 04 §8 is whole minutes). Each job has a
running flag; errors are logged.

- Retention (at start and when the date changes, once AMI is up):
  recordings, voicemail past voicemail.retentionDays (VoicemailRemove, so
  Asterisk renumbers and updates MWI), expired sessions, webhook
  deliveries older than 30 days.
- publicIpWatch (every 5 min, Settings -> Network): a changed IP is saved,
  audited, applied (restart required) and emailed to admins.
- trunkWatch: a registering trunk rejected/unregistered for 5 minutes ->
  one email per outage.
- diskWatch (hourly): under 10% free -> one email a day.
- A failed daily backup emails admins too.

Settings gets a Network tab (public IP, Detect, re-check) and voicemail
retention on the Recording tab.

Task: M11-T4

apps/api/src/index.ts

@@ -17,10 +17,7 @@ import { watchToggles } from './services/toggles.ts';
 import { watchCdr } from './services/cdr.ts';
 import { watchQueueCalls } from './services/queuestats.ts';
 import { watchPinChanges } from './services/vmpin.ts';
-import { cleanupRecordings } from './routes/recordings.ts';
-import { getSetting } from './services/settings.ts';
-import { todayIn } from './generator/load.ts';
-import { backupDue, runBackup } from './services/backup.ts';
+import { startJobs } from './jobs.ts';
 
 let config;
 try {
@@ -108,42 +105,21 @@ ami.on('connected', () => {
 });
 ami.start();
 
-// Holidays roll over at midnight (03 §6.10): render again when the local date changes.
-// ponytail: a once-a-minute check instead of croner; the M11 jobs can take this over.
-let lastDay = '';
-const regenerateOnNewDay = async () => {
-  const today = todayIn(await getSetting(db, 'general.timezone'));
-  if (lastDay && today !== lastDay) await generator.apply({ trigger: 'nightly', force: true });
-  if (today !== lastDay) {
-    // Once a day (and at start): recordings past their retention (04 §8, 08).
-    const removed = await cleanupRecordings(
-      db,
-      resolve(config.RECORDINGS_DIR),
-      await getSetting(db, 'recordings.retentionDays'),
-    );
-    if (removed) app.log.info({ removed }, 'old recordings deleted');
-  }
-  lastDay = today;
-};
-/** The daily backup at backup.time (04 §8.1). A failure shows on the dashboard (backup_old). */
-let backupTriedDay = ''; // one try a day: a failing pg_dump shouldn't run every minute
-const backupIfDue = async () => {
-  const today = todayIn(await getSetting(db, 'general.timezone'));
-  if (backupTriedDay === today || !(await backupDue(config, db))) return;
-  backupTriedDay = today;
-  const b = await runBackup(config, db);
-  app.log.info({ backup: b.name, size: b.size }, 'backup made');
-};
-void regenerateOnNewDay();
-const dayTimer = setInterval(() => {
-  void regenerateOnNewDay().catch((err) => app.log.error({ err }, 'nightly regenerate failed'));
-  void backupIfDue().catch((err) => app.log.error({ err }, 'backup failed'));
-}, 60_000);
+const stopJobs = startJobs({
+  config,
+  db,
+  ami,
+  live,
+  generator,
+  voicemail,
+  send: sendMail,
+  log: app.log,
+});
 
 for (const sig of ['SIGTERM', 'SIGINT'] as const) {
   process.once(sig, () => {
     generator.cancelPending();
-    clearInterval(dayTimer);
+    stopJobs();
     history.stop();
     ami.stop();
     void app

apps/api/src/jobs.ts

@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Background jobs (04 §8) on one once-a-minute timer.
+// ponytail: setInterval instead of croner: every schedule in 04 §8 is a whole number of minutes.
+import { statfs } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { and, eq, isNull, lt, sql } from 'drizzle-orm';
+import { z } from 'zod';
+import type { Config } from './config.ts';
+import type { Db } from './db/client.ts';
+import {
+  extensions,
+  sessions,
+  trunks,
+  users,
+  voicemailMessages,
+  webhookDeliveries,
+} from './db/schema.ts';
+import type { Generator } from './generator/index.ts';
+import { todayIn } from './generator/load.ts';
+import type { LiveState } from './live/state.ts';
+import { cleanupRecordings } from './routes/recordings.ts';
+import { audit } from './services/audit.ts';
+import { backupDue, runBackup } from './services/backup.ts';
+import type { SendMail } from './services/mail.ts';
+import { getSetting, setSetting } from './services/settings.ts';
+
+export type JobDeps = {
+  config: Config;
+  db: Db;
+  ami: { isConnected(): boolean; action(f: Record<string, string>): Promise<unknown> };
+  live: Pick<LiveState, 'snapshot' | 'amiConnected'>;
+  generator: Pick<Generator, 'apply'>;
+  voicemail: { scan(ext: string): Promise<void> };
+  send: SendMail;
+  log: { info(o: object, m: string): void; error(o: object, m: string): void };
+  now?: () => Date;
+  statfs?: typeof statfs;
+};
+
+/** The server's public IPv4 as an outside service sees it, or null. */
+export async function detectPublicIp(): Promise<string | null> {
+  for (const url of ['https://api.ipify.org', 'https://ifconfig.me/ip']) {
+    const ip = await fetch(url, { signal: AbortSignal.timeout(5000) })
+      .then((r) => (r.ok ? r.text() : ''))
+      .catch(() => '');
+    if (z.ipv4().safeParse(ip.trim()).success) return ip.trim();
+  }
+  return null;
+}
+
+/** Email every active admin (08 §3.2). Email not set up or failing is logged, not thrown. */
+export async function mailAdmins(d: JobDeps, subject: string, text: string) {
+  const to = (
+    await d.db
+      .select({ email: users.email })
+      .from(users)
+      .where(and(eq(users.role, 'admin'), eq(users.isActive, true)))
+  ).map((r) => r.email);
+  if (to.length)
+    await d.send({ to, subject, text }).catch((err: unknown) => d.log.error({ err }, subject));
+}
+
+/** Recordings, voicemail, expired sessions and old webhook deliveries (04 §8 retentionCleanup). */
+export async function retentionCleanup(d: JobDeps) {
+  const recordings = await cleanupRecordings(
+    d.db,
+    resolve(d.config.RECORDINGS_DIR),
+    await getSetting(d.db, 'recordings.retentionDays'),
+  );
+  let voicemail = 0;
+  const vmDays = await getSetting(d.db, 'voicemail.retentionDays');
+  // Asterisk removes the files and renumbers the folder (the caller waits for AMI).
+  if (vmDays > 0) {
+    const old = await d.db
+      .select({
+        ext: extensions.number,
+        folder: voicemailMessages.folder,
+        msgId: voicemailMessages.msgId,
+      })
+      .from(voicemailMessages)
+      .innerJoin(extensions, eq(extensions.id, voicemailMessages.extensionId))
+      .where(
+        and(
+          isNull(voicemailMessages.deletedAt),
+          lt(voicemailMessages.receivedAt, sql`now() - make_interval(days => ${vmDays})`),
+        ),
+      );
+    for (const m of old)
+      await d.ami.action({
+        Action: 'VoicemailRemove',
+        Context: 'default',
+        Mailbox: m.ext,
+        Folder: m.folder,
+        ID: m.msgId,
+      });
+    for (const ext of new Set(old.map((m) => m.ext))) await d.voicemail.scan(ext);
+    voicemail = old.length;
+  }
+  await d.db.delete(sessions).where(lt(sessions.expiresAt, sql`now()`));
+  await d.db
+    .delete(webhookDeliveries)
+    .where(lt(webhookDeliveries.createdAt, sql`now() - interval '30 days'`));
+  if (recordings || voicemail) d.log.info({ recordings, voicemail }, 'old messages deleted');
+}
+
+/** The jobs, each keeping its own state (tests call them directly). */
+export function createJobs(d: JobDeps) {
+  const now = d.now ?? (() => new Date());
+  const tz = () => getSetting(d.db, 'general.timezone');
+
+  // Holidays roll over at midnight (03 §6.10): render again when the local date changes.
+  // Retention runs then too (and at start), not at 02:00.
+  let lastDay = '';
+  let cleanedDay = '';
+  const daily = async () => {
+    const today = todayIn(await tz(), now());
+    if (lastDay && today !== lastDay) await d.generator.apply({ trigger: 'nightly', force: true });
+    lastDay = today;
+    // Voicemail goes through Asterisk: at start, wait the few seconds until AMI is connected.
+    if (cleanedDay !== today && d.ami.isConnected()) {
+      await retentionCleanup(d);
+      cleanedDay = today;
+    }
+  };
+
+  let backupTriedDay = ''; // one try a day: a failing pg_dump shouldn't run every minute
+  const backup = async () => {
+    const today = todayIn(await tz(), now());
+    if (backupTriedDay === today || !(await backupDue(d.config, d.db, now()))) return;
+    backupTriedDay = today;
+    try {
+      const b = await runBackup(d.config, d.db);
+      d.log.info({ backup: b.name, size: b.size }, 'backup made');
+    } catch (err) {
+      await mailAdmins(d, 'Backup failed', `Today's backup failed: ${(err as Error).message}`);
+      throw err;
+    }
+  };
+
+  const publicIpWatch = async () => {
+    if (!(await getSetting(d.db, 'network.publicIpWatch'))) return;
+    const ip = await detectPublicIp();
+    const old = await getSetting(d.db, 'network.publicIp');
+    if (!ip || ip === old) return;
+    await setSetting(d.db, 'network.publicIp', ip);
+    await audit(d.db, {
+      action: 'settings',
+      summary: `Public IP changed from ${old ?? 'none'} to ${ip}`,
+      diff: { before: old, after: ip },
+    });
+    await d.generator.apply({ trigger: 'publicIp', force: true }); // sets restartRequired
+    await mailAdmins(
+      d,
+      `Public IP changed to ${ip}`,
+      `The phone system's public IP changed from ${old ?? 'none'} to ${ip}.\n` +
+        'Restart telephony from the dashboard so calls use the new address (it drops calls in progress).\n' +
+        'Phones and the provider may need the new IP if they were set up with the old one.',
+    );
+  };
+
+  // A registering trunk that stays down for 5 minutes: one email per outage.
+  const downSince = new Map<string, number>();
+  const mailed = new Set<string>();
+  const trunkWatch = async () => {
+    if (!d.live.amiConnected) return; // Asterisk itself being down shows elsewhere
+    const state = new Map((d.live.snapshot(true).trunks ?? []).map((t) => [t.trunkId, t]));
+    const rows = await d.db
+      .select({ id: trunks.id, name: trunks.name })
+      .from(trunks)
+      .where(eq(trunks.enabled, true));
+    for (const t of rows) {
+      const reg = state.get(t.id)?.registration;
+      // ponytail: IP-auth trunks don't register (always 'unknown'), so they're never "down" here.
+      if (reg !== 'rejected' && reg !== 'unregistered') {
+        downSince.delete(t.id);
+        mailed.delete(t.id);
+        continue;
+      }
+      const since = downSince.get(t.id) ?? now().getTime();
+      downSince.set(t.id, since);
+      if (now().getTime() - since < 5 * 60_000 || mailed.has(t.id)) continue;
+      mailed.add(t.id);
+      await mailAdmins(
+        d,
+        `Trunk ${t.name} is down`,
+        `Trunk ${t.name} has not been registered with the provider since ` +
+          `${new Date(since).toISOString()} (${reg}). Calls through it fail.\n` +
+          'Check Trunks → Test, the account password, and the provider status.',
+      );
+    }
+  };
+
+  let diskMailedDay = '';
+  const diskWatch = async () => {
+    const fs = await (d.statfs ?? statfs)(resolve(d.config.RECORDINGS_DIR));
+    const free = fs.bavail / fs.blocks;
+    const today = todayIn(await tz(), now());
+    if (free >= 0.1 || diskMailedDay === today) return;
+    diskMailedDay = today;
+    await mailAdmins(
+      d,
+      `Disk almost full (${Math.round(free * 100)}% free)`,
+      `Only ${Math.round((fs.bavail * fs.bsize) / 1e9)} GB (${Math.round(free * 100)}%) is free ` +
+        'on the phone system. Recordings, voicemail and backups stop being saved when it fills.\n' +
+        'Lower the recording retention (Settings → Recording) or free up space.',
+    );
+  };
+
+  return { daily, backup, publicIpWatch, trunkWatch, diskWatch };
+}
+
+/** Starts the jobs; returns a stop function. */
+export function startJobs(d: JobDeps): () => void {
+  const j = createJobs(d);
+  // Every job: never two runs of itself at once (04 §8), errors logged.
+  const running = new Set<string>();
+  const run = (name: string, fn: () => Promise<void>) => {
+    if (running.has(name)) return;
+    running.add(name);
+    void fn()
+      .catch((err: unknown) => d.log.error({ err }, `job ${name} failed`))
+      .finally(() => running.delete(name));
+  };
+  let minute = 0;
+  const tick = () => {
+    run('daily', j.daily);
+    run('backup', j.backup);
+    run('trunkWatch', j.trunkWatch);
+    if (minute % 5 === 0) run('publicIpWatch', j.publicIpWatch);
+    if (minute % 60 === 0) run('diskWatch', j.diskWatch);
+    minute++;
+  };
+  tick();
+  const timer = setInterval(tick, 60_000);
+  return () => clearInterval(timer);
+}

apps/api/src/routes/settings.ts

@@ -22,6 +22,7 @@ import { getAllSettings, getSetting, setSetting } from '../services/settings.ts'
 import { notifyEmergency } from '../services/emergency.ts';
 import { readTlsCert } from '../services/tls.ts';
 import { backupMissed } from '../services/backup.ts';
+import { detectPublicIp } from '../jobs.ts';
 import type { SendMail } from '../services/mail.ts';
 
 declare module 'fastify' {
@@ -179,12 +180,8 @@ export async function settingsRoutes(app: FastifyInstance) {
 
   /** The server's public IPv4, as an outside service sees it (only sent when asked). */
   app.post('/api/settings/network/detect-public-ip', async () => {
-    for (const url of ['https://api.ipify.org', 'https://ifconfig.me/ip']) {
-      const ip = await fetch(url, { signal: AbortSignal.timeout(5000) })
-        .then((r) => (r.ok ? r.text() : ''))
-        .catch(() => '');
-      if (z.ipv4().safeParse(ip.trim()).success) return { ip: ip.trim() };
-    }
+    const ip = await detectPublicIp();
+    if (ip) return { ip };
     throw new HttpError(502, 'unavailable', "Couldn't detect the public IP. Enter it by hand.");
   });
 

apps/api/test/jobs.test.ts

@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// M11-T4: background jobs (04 §8): retention, trunk/disk/public IP watches.
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import type { LightMyRequestResponse } from 'fastify';
+import { eq } from 'drizzle-orm';
+import type { TrunkLive } from '@selfphoneng/shared';
+import { createJobs, retentionCleanup, type JobDeps } from '../src/jobs.ts';
+import { extensions, sessions, voicemailMessages } from '../src/db/schema.ts';
+import { getSetting, setSetting } from '../src/services/settings.ts';
+import { hasTestDb } from './db.ts';
+import { ADMIN, H, createTestApp } from './app.ts';
+
+describe.skipIf(!hasTestDb)('background jobs', () => {
+  let t: Awaited<ReturnType<typeof createTestApp>>;
+  let req: (m: string, url: string, payload?: unknown) => Promise<LightMyRequestResponse>;
+  let trunkId: string;
+  let clock = Date.parse('2026-10-19T12:00:00Z');
+  let reg: TrunkLive['registration'] = 'registered';
+  const scanned: string[] = [];
+  const deps = (): JobDeps => ({
+    config: t.app.config,
+    db: t.db,
+    ami: t.ami,
+    live: {
+      amiConnected: true,
+      snapshot: () => ({
+        extensions: [],
+        queues: [],
+        parked: [],
+        ami: { connected: true },
+        trunks: [{ trunkId, registration: reg, reachable: true, lastChange: '' }],
+      }),
+    },
+    generator: t.app.generator,
+    voicemail: { scan: async (ext) => void scanned.push(ext) },
+    send: t.app.sendMail,
+    log: t.app.log,
+    now: () => new Date(clock),
+  });
+
+  beforeAll(async () => {
+    t = await createTestApp();
+    await t.clearMustChange(ADMIN.email);
+    const cookie = await t.login(ADMIN.email, ADMIN.password);
+    req = (method, url, payload) =>
+      t.app.inject({
+        method: method as 'GET',
+        url,
+        payload: payload as object,
+        headers: { ...H, cookie },
+      });
+    await req('POST', '/api/extensions', { number: '101', name: 'Dan' });
+    const r = await req('POST', '/api/trunks', {
+      name: 'voip.ms',
+      serverHost: '127.0.0.1',
+      username: '123456_pbx',
+      password: 'Sub-account pw!',
+    });
+    trunkId = r.json().id;
+  });
+  afterAll(() => t.close());
+  afterEach(() => vi.unstubAllGlobals());
+
+  it('retention removes old voicemail through Asterisk, and expired sessions', async () => {
+    const [ext] = await t.db.select().from(extensions).where(eq(extensions.number, '101'));
+    const msg = (msgId: string, days: number) => ({
+      extensionId: ext!.id,
+      msgId,
+      folder: 'INBOX',
+      durationS: 3,
+      receivedAt: new Date(Date.now() - days * 86_400_000),
+      filePath: `default/101/INBOX/${msgId}.wav`,
+    });
+    await t.db.insert(voicemailMessages).values([msg('old-1', 40), msg('new-1', 2)]);
+    const [u] = await t.db.select({ id: sessions.userId }).from(sessions);
+    await t.db.insert(sessions).values({
+      id: 'expired-session-hash',
+      userId: u!.id,
+      csrfToken: 'x',
+      expiresAt: new Date(Date.now() - 1000),
+    } as typeof sessions.$inferInsert);
+
+    await setSetting(t.db, 'voicemail.retentionDays', 0); // 0 keeps voicemail
+    t.ami.actions.length = 0;
+    await retentionCleanup(deps());
+    expect(t.ami.actions).toEqual([]);
+    expect(
+      await t.db.select().from(sessions).where(eq(sessions.id, 'expired-session-hash')),
+    ).toEqual([]);
+
+    await setSetting(t.db, 'voicemail.retentionDays', 30);
+    await retentionCleanup(deps());
+    expect(t.ami.actions).toEqual([
+      {
+        Action: 'VoicemailRemove',
+        Context: 'default',
+        Mailbox: '101',
+        Folder: 'INBOX',
+        ID: 'old-1',
+      },
+    ]);
+    expect(scanned).toEqual(['101']);
+
+    // The daily job waits for Asterisk (at start AMI connects a few seconds after the first run).
+    await t.db.update(voicemailMessages).set({ deletedAt: null }); // fake scan: still there
+    const j = createJobs(deps());
+    t.ami.connected = false;
+    t.ami.actions.length = 0;
+    await j.daily();
+    expect(t.ami.actions).toEqual([]);
+    t.ami.connected = true;
+    await j.daily();
+    await j.daily(); // once a day
+    expect(t.ami.actions.map((a) => a.ID)).toEqual(['old-1']);
+  });
+
+  it('trunkWatch emails admins once when a trunk is down for 5 minutes', async () => {
+    const j = createJobs(deps());
+    t.mails.length = 0;
+    reg = 'rejected';
+    await j.trunkWatch();
+    clock += 4 * 60_000;
+    await j.trunkWatch();
+    expect(t.mails).toEqual([]);
+    clock += 60_000;
+    await j.trunkWatch();
+    clock += 60_000;
+    await j.trunkWatch();
+    expect(t.mails.map((m) => [m.to, m.subject])).toEqual([
+      [[ADMIN.email], 'Trunk voip.ms is down'],
+    ]);
+    // Back up, then down again: a new outage gets its own email.
+    reg = 'registered';
+    await j.trunkWatch();
+    reg = 'unregistered';
+    await j.trunkWatch();
+    clock += 5 * 60_000;
+    await j.trunkWatch();
+    expect(t.mails).toHaveLength(2);
+    // IP-auth trunks never register: never "down".
+    reg = 'unknown';
+    const k = createJobs(deps());
+    await k.trunkWatch();
+    clock += 10 * 60_000;
+    await k.trunkWatch();
+    expect(t.mails).toHaveLength(2);
+  });
+
+  it('publicIpWatch saves a changed IP, asks for a restart and emails admins', async () => {
+    const j = createJobs(deps());
+    vi.stubGlobal('fetch', async () => new Response('203.0.113.9\n'));
+    t.mails.length = 0;
+    await setSetting(t.db, 'network.publicIp', '203.0.113.5');
+    await j.publicIpWatch(); // off by default
+    expect(await getSetting(t.db, 'network.publicIp')).toBe('203.0.113.5');
+
+    await setSetting(t.db, 'network.publicIpWatch', true);
+    await j.publicIpWatch();
+    expect(await getSetting(t.db, 'network.publicIp')).toBe('203.0.113.9');
+    expect(await getSetting(t.db, 'system.restartRequired')).toBe(true);
+    expect(t.mails.map((m) => m.subject)).toEqual(['Public IP changed to 203.0.113.9']);
+
+    await j.publicIpWatch(); // unchanged: nothing
+    vi.stubGlobal('fetch', async () => new Response('<html>rate limited</html>'));
+    await j.publicIpWatch(); // garbage answer: keep the old IP
+    expect(await getSetting(t.db, 'network.publicIp')).toBe('203.0.113.9');
+    expect(t.mails).toHaveLength(1);
+  });
+
+  it('diskWatch emails admins once a day under 10% free', async () => {
+    let bavail = 50;
+    const fake = (async () => ({
+      bavail,
+      blocks: 100,
+      bsize: 1e9,
+    })) as unknown as JobDeps['statfs'];
+    const j = createJobs({ ...deps(), statfs: fake });
+    t.mails.length = 0;
+    await j.diskWatch();
+    expect(t.mails).toEqual([]);
+    bavail = 6;
+    await j.diskWatch();
+    await j.diskWatch();
+    expect(t.mails.map((m) => [m.subject, m.text.split('.')[0]])).toEqual([
+      ['Disk almost full (6% free)', 'Only 6 GB (6%) is free on the phone system'],
+    ]);
+    clock += 86_400_000; // next day: again
+    await j.diskWatch();
+    expect(t.mails).toHaveLength(2);
+  });
+
+  it('a mail that fails to send is logged, not thrown', async () => {
+    t.mail.fail = 'smtp down';
+    reg = 'rejected';
+    const j = createJobs(deps());
+    await j.trunkWatch();
+    clock += 6 * 60_000;
+    await expect(j.trunkWatch()).resolves.toBeUndefined();
+    t.mail.fail = null;
+  });
+});

apps/web/src/pages/admin/SettingsPage.tsx

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-// Settings (05 §2). M3: Email and Emergency; M7: Recording; M11: Security (phone ACL, provisioning, SIP TLS), Backups.
+// Settings (05 §2). M3: Email and Emergency; M7: Recording; M11: Network (public IP watch), Security (phone ACL, provisioning, SIP TLS), Backups.
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { useState } from 'react';
 import { NavLink, useParams } from 'react-router';
@@ -12,6 +12,7 @@ const TABS = [
   { id: 'email', label: 'Email' },
   { id: 'emergency', label: 'Emergency' },
   { id: 'recording', label: 'Recording' },
+  { id: 'network', label: 'Network' },
   { id: 'security', label: 'Security' },
   { id: 'backups', label: 'Backups' },
 ] as const;
@@ -42,6 +43,7 @@ export function SettingsPage() {
       {settings.data && tab === 'email' && <EmailTab s={settings.data} />}
       {settings.data && tab === 'emergency' && <EmergencyTab s={settings.data} />}
       {settings.data && tab === 'recording' && <RecordingTab s={settings.data} />}
+      {settings.data && tab === 'network' && <NetworkTab s={settings.data} />}
       {settings.data && tab === 'security' && <SecurityTab s={settings.data} />}
       {settings.data && tab === 'backups' && <BackupsTab s={settings.data} />}
     </section>
@@ -322,6 +324,7 @@ export function EmergencyTab({ s }: { s: SettingsDto }) {
 function RecordingTab({ s }: { s: SettingsDto }) {
   const save = useSave();
   const [days, setDays] = useState(String(s['recordings.retentionDays']));
+  const [vmDays, setVmDays] = useState(String(s['voicemail.retentionDays']));
   const [notice, setNotice] = useState<string | null>(s['recordings.announcementSoundId']);
   return (
     <form
@@ -330,6 +333,7 @@ function RecordingTab({ s }: { s: SettingsDto }) {
         e.preventDefault();
         save.mutate({
           'recordings.retentionDays': Number(days),
+          'voicemail.retentionDays': Number(vmDays),
           'recordings.announcementSoundId': notice,
         });
       }}
@@ -367,6 +371,17 @@ function RecordingTab({ s }: { s: SettingsDto }) {
           />
         )}
       </Field>
+      <Field label="Keep voicemail for (days)" hint="0 keeps it until the user deletes it">
+        {(p) => (
+          <input
+            {...p}
+            className={`${inputClass} max-w-32`}
+            inputMode="numeric"
+            value={vmDays}
+            onChange={(e) => setVmDays(e.target.value)}
+          />
+        )}
+      </Field>
       <div className="flex items-center gap-3">
         <Button type="submit" variant="primary" disabled={save.isPending}>
           Save
@@ -378,6 +393,57 @@ function RecordingTab({ s }: { s: SettingsDto }) {
   );
 }
 
+/** The public IP Asterisk tells callers about, and re-checking it on a dynamic IP (01 §4.3, 04 §8). */
+function NetworkTab({ s }: { s: SettingsDto }) {
+  const save = useSave();
+  const [ip, setIp] = useState(s['network.publicIp'] ?? '');
+  const [watch, setWatch] = useState(s['network.publicIpWatch']);
+  const detect = useMutation({ mutationFn: systemApi.detectPublicIp, onSuccess: setIp });
+  return (
+    <form
+      className="flex flex-col gap-4"
+      onSubmit={(e) => {
+        e.preventDefault();
+        save.mutate({ 'network.publicIp': ip.trim() || null, 'network.publicIpWatch': watch });
+      }}
+    >
+      <Field
+        label="Public IP"
+        hint="Changing it needs a telephony restart (the dashboard offers one)"
+      >
+        {(p) => (
+          <div className="flex gap-2">
+            <input
+              {...p}
+              className={`${inputClass} max-w-48`}
+              value={ip}
+              onChange={(e) => setIp(e.target.value)}
+            />
+            <Button type="button" onClick={() => detect.mutate()} disabled={detect.isPending}>
+              Detect
+            </Button>
+          </div>
+        )}
+      </Field>
+      <label className="flex items-center gap-2 text-sm">
+        <input type="checkbox" checked={watch} onChange={(e) => setWatch(e.target.checked)} />
+        Re-check the public IP every 5 minutes (dynamic IP)
+      </label>
+      <p className="-mt-2 text-xs text-zinc-500">
+        When it changes, the new IP is saved and admins get an email asking for a telephony restart.
+        Asks api.ipify.org (or ifconfig.me) for the address.
+      </p>
+      <div className="flex items-center gap-3">
+        <Button type="submit" variant="primary" disabled={save.isPending}>
+          Save
+        </Button>
+        {save.isSuccess && <span className="text-sm text-zinc-500">Saved.</span>}
+      </div>
+      {(save.error ?? detect.error) && <Alert>{(save.error ?? detect.error)!.message}</Alert>}
+    </form>
+  );
+}
+
 /** Phone ACL (08 §2.2), provisioning access (07 §2) and SIP over TLS (M11-T2). */
 function SecurityTab({ s }: { s: SettingsDto }) {
   const save = useSave();

docs/spec/04-backend-api.md

@@ -447,6 +447,22 @@ Provisioning and inbound hooks (outside `/api`, no session):
 
 Jobs must never overlap with themselves (use a running flag).
 
+As built (M11-T4): `jobs.ts`, one setInterval a minute instead of croner (every schedule above is
+whole minutes). Each job has a running flag; an error is logged, not fatal.
+- `nightlyRegenerate` and `retentionCleanup` run when the date in `general.timezone` changes and
+  at start, not at 00:05 / 02:00. Retention waits for AMI: voicemail is removed with
+  `VoicemailRemove`, so Asterisk renumbers the folder and updates MWI. It also deletes recordings,
+  expired sessions and webhook deliveries older than 30 days.
+- `publicIpWatch` asks api.ipify.org (then ifconfig.me), only when the setting is on (Settings →
+  Network). A changed IP is saved, audited, applied (which sets `restartRequired`; the job doesn't
+  restart Asterisk, that drops calls) and emailed.
+- `trunkWatch` only knows registering trunks (IP-auth trunks are always `unknown`): rejected or
+  unregistered for 5 minutes in a row → one email; registered again ends the outage. Skipped while
+  AMI is down. No "back up" email.
+- `diskWatch` checks the filesystem of `RECORDINGS_DIR` (`statfs`).
+- Admin emails go to every active admin; the texts are in `jobs.ts`, not `mail.ts`. A failed daily
+  backup emails them too (08 §3.2). Email not set up: logged, nothing else.
+
 ### 8.1 Backups
 Folder `/var/lib/selfphoneng/backups/selfphoneng-YYYYMMDD-HHMM.tar.gz` containing:
 `db.sql` (`pg_dump --no-owner`), `voicemail/`, `sounds-custom/`, `moh/`,

scripts/e2e/README.md

@@ -227,3 +227,12 @@ user Dan with 201, a trunk with a password, voicemail, a sound, music on hold, a
 `APP_SECRET` and on a "no", then restores it: the admin signs in with A's password, 201 and the
 trunk are back, the trunk password decrypts into Asterisk's config, the files match byte for byte.
 `KEEP=1` leaves both servers running.
+
+`m11-t4.mjs` (m2-live started with a voicemail dir, after `m1-t4.mjs`; stop your api first: the
+script starts and restarts it, with the api's environment taken from its own) is the M11-T4
+acceptance, with mail going to a local SMTP sink on 127.0.0.1:2525. Settings → Network turns on
+the public IP watch and Settings → Recording keeps voicemail 30 days. A real voicemail, made 40
+days old in the database, is removed through Asterisk (files gone) when the api restarts. The
+watch finds the real public IP (needs internet), saves it, asks for a restart and emails the
+admins. A trunk to 192.0.2.1 (nothing answers) gets exactly one "is down" email after 5 minutes,
+and a disk under 10% free gets a "Disk almost full" email. About 8 minutes.

scripts/e2e/m11-t4.mjs

@@ -0,0 +1,211 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// M11-T4: the background jobs against a real Asterisk, with email to a local SMTP sink.
+// - Settings → Network turns on the public IP watch, Settings → Recording sets voicemail retention.
+// - A real voicemail, made 40 days old in the database, is removed through Asterisk when the api
+//   starts (and the message files are gone).
+// - The watch finds the real public IP (needs internet), saves it and emails the admins.
+// - A trunk to a server that never answers is down for 5 minutes: one email (takes ~6 minutes).
+// - Disk under 10% free emails the admins, otherwise nothing.
+// Runs on the m2-live setup started with a voicemail dir, after m1-t4.mjs. This script starts and
+// restarts the api itself (stop yours first), with the api's environment in its own:
+//   DATABASE_URL AMI_PORT=15038 AMI_SECRET=m2-live-secret ASTERISK_CONFIG_DIR VOICEMAIL_DIR
+//   API_PORT, ... (and the repo's .env for the rest)
+// Usage: node scripts/e2e/m11-t4.mjs <adminEmail> <adminPassword> <shotsDir>
+import assert from 'node:assert/strict';
+import { spawn, spawnSync } from 'node:child_process';
+import { createWriteStream, existsSync, statfsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { launch } from './cdp.mjs';
+import { smtpSink } from './smtp-sink.mjs';
+
+const AST = 'sp-live-ast';
+const EXT = '201';
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const docker = (...a) => {
+  const r = spawnSync('docker', a, { encoding: 'utf8' });
+  return { ok: r.status === 0, out: `${r.stdout ?? ''}${r.stderr ?? ''}` };
+};
+const cli = (cmd) => docker('exec', AST, 'asterisk', '-rx', cmd).out;
+const sql = (q) =>
+  docker(
+    'exec',
+    'selfphoneng-postgres-1',
+    'psql',
+    '-U',
+    'selfphoneng',
+    '-d',
+    new URL(process.env.DATABASE_URL).pathname.slice(1),
+    '-tAc',
+    q,
+  ).out.trim();
+const until = async (what, ms, fn) => {
+  const end = Date.now() + ms;
+  while (!(await fn())) {
+    assert.ok(Date.now() < end, `timed out: ${what}`);
+    await sleep(1000);
+  }
+};
+const [email, pw, shotsDir] = process.argv.slice(2);
+const base = `http://127.0.0.1:${process.env.API_PORT}`;
+const repo = resolve(import.meta.dirname, '../..');
+const box = join(process.env.VOICEMAIL_DIR, 'default', EXT, 'INBOX');
+
+const { mails, server: smtp } = await smtpSink();
+const mailed = (subject) => mails.some((m) => m.includes(`Subject: ${subject}`));
+
+const log = createWriteStream(join(shotsDir, 'm11-t4-api.log'));
+let apiProc;
+const startApi = async () => {
+  apiProc = spawn(
+    process.execPath,
+    [`--env-file=${repo}/.env`, '--conditions=source', `${repo}/apps/api/src/index.ts`],
+    { stdio: ['ignore', 'pipe', 'pipe'] }, // cwd: where the web build is (./public)
+  );
+  apiProc.stdout.pipe(log, { end: false });
+  apiProc.stderr.pipe(log, { end: false });
+  await until('api up', 30_000, () =>
+    fetch(`${base}/healthz`).then(
+      (r) => r.ok,
+      () => false,
+    ),
+  );
+};
+const stopApi = async () => {
+  const gone = new Promise((r) => apiProc.once('exit', r));
+  apiProc.kill();
+  await gone;
+};
+
+let cookie;
+const api = async (method, url, body) => {
+  const r = await fetch(`${base}${url}`, {
+    method,
+    headers: {
+      'x-requested-with': 'selfphoneng',
+      cookie,
+      ...(body ? { 'content-type': 'application/json' } : {}),
+    },
+    body: body && JSON.stringify(body),
+  });
+  const json = await r.json();
+  assert.ok(r.ok, `${method} ${url}: ${r.status} ${JSON.stringify(json)}`);
+  return json;
+};
+const signIn = async () => {
+  const r = await fetch(`${base}/api/auth/login`, {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', 'x-requested-with': 'selfphoneng' },
+    body: JSON.stringify({ email, password: pw }),
+  });
+  assert.equal(r.status, 200, 'login');
+  cookie = r.headers
+    .getSetCookie()
+    .map((c) => c.split(';')[0])
+    .join('; ');
+};
+
+await startApi();
+const p = await launch({ shotsDir });
+try {
+  await signIn();
+  await api('PATCH', '/api/settings', {
+    smtp: {
+      host: '127.0.0.1',
+      port: 2525,
+      secure: false,
+      username: '',
+      from: 'PBX <pbx@example.com>',
+    },
+  });
+  await api('POST', '/api/extensions', { number: EXT, name: 'Ann Lee' });
+  // TEST-NET-1: nothing answers, so the registration never succeeds.
+  await api('POST', '/api/trunks', {
+    name: 'dead-provider',
+    serverHost: '192.0.2.1',
+    username: '999999_pbx',
+    password: 'never-used-pw-1',
+  });
+
+  // The settings, in the browser.
+  await p.goto(`${base}/login`);
+  await p.waitFor(`!!document.querySelector('input[type=email]')`);
+  await p.fill('Email', email);
+  await p.fill('Password', pw);
+  await p.click('Sign in');
+  await p.waitFor(`location.pathname.startsWith('/admin')`, 'signed in');
+  await p.goto(`${base}/admin/settings/network`);
+  await p.waitFor(`document.body.innerText.includes('Re-check the public IP')`, 'network tab');
+  await p.fill('Public IP', '203.0.113.5');
+  await p.evaluate(
+    `[...document.querySelectorAll('label')].find((l) => l.innerText.includes('Re-check the public IP')).querySelector('input').click()`,
+  );
+  await p.click('Save');
+  await p.waitFor(`document.body.innerText.includes('Saved.')`, 'network saved');
+  await p.shot('m11-t4-01-network');
+  await p.goto(`${base}/admin/settings/recording`);
+  await p.waitFor(`document.body.innerText.includes('Keep voicemail for')`, 'recording tab');
+  await p.fill('Keep voicemail for (days)', '30');
+  await p.click('Save');
+  await p.waitFor(`document.body.innerText.includes('Saved.')`, 'retention saved');
+  const s = await api('GET', '/api/settings');
+  assert.equal(s['network.publicIp'], '203.0.113.5');
+  assert.equal(s['network.publicIpWatch'], true);
+  assert.equal(s['voicemail.retentionDays'], 30);
+  console.log('settings: public IP watch on, voicemail kept 30 days (saved in the browser)');
+
+  // A real voicemail (Local channel waits out the greeting, then "speaks"), made old.
+  cli(
+    `channel originate Local/${EXT}@sp-vm-direct application Playback silence/10&silence/5&tt-weasels`,
+  );
+  await until(
+    'voicemail saved',
+    90_000,
+    () => sql('select count(*) from voicemail_messages') === '1',
+  );
+  assert.ok(existsSync(join(box, 'msg0000.wav')), 'message file on disk');
+  sql(`update voicemail_messages set received_at = now() - interval '40 days'`);
+  mails.length = 0; // the voicemail email
+  console.log('voicemail left and made 40 days old');
+
+  // Restart: the jobs run at start.
+  await stopApi();
+  await startApi();
+  await until('old voicemail removed', 90_000, () => !existsSync(join(box, 'msg0000.txt')));
+  assert.ok(!existsSync(join(box, 'msg0000.wav')), 'audio removed too');
+  await until(
+    'row marked deleted',
+    10_000,
+    () => sql('select count(*) from voicemail_messages where deleted_at is null') === '0',
+  );
+  console.log('retention: Asterisk removed the 40-day-old voicemail');
+
+  await until('public IP email', 60_000, () => mailed('Public IP changed to'));
+  await signIn();
+  const ip = (await api('GET', '/api/settings'))['network.publicIp'];
+  assert.notEqual(ip, '203.0.113.5');
+  assert.ok(mailed(`Public IP changed to ${ip}`));
+  const warnings = (await api('GET', '/api/system/warnings')).items.map((w) => w.code);
+  assert.ok(warnings.includes('restart_required'), `restart asked for: ${warnings}`);
+  console.log(`public IP watch: found ${ip}, saved, restart required, admins emailed`);
+
+  const disk = statfsSync(process.env.RECORDINGS_DIR ?? process.env.VOICEMAIL_DIR);
+  const low = disk.bavail / disk.blocks < 0.1;
+  await until('disk check', 10_000, () => !low || mailed('Disk almost full'));
+  if (!low) assert.ok(!mailed('Disk almost full'));
+  console.log(
+    `disk watch: ${Math.round((100 * disk.bavail) / disk.blocks)}% free, ${low ? '' : 'no '}email`,
+  );
+
+  assert.ok(!mailed('Trunk dead-provider is down'), 'not before 5 minutes');
+  assert.ok(!/dead-provider.*Registered/.test(cli('pjsip show registrations')));
+  await until('trunk down email', 8 * 60_000, () => mailed('Trunk dead-provider is down'));
+  await sleep(65_000); // one more check: still one email
+  assert.equal(mails.filter((m) => m.includes('Subject: Trunk dead-provider is down')).length, 1);
+  console.log('trunk watch: one email after 5 minutes down');
+  console.log('M11-T4: PASS');
+} finally {
+  await p.close();
+  await stopApi();
+  smtp.close();
+  log.end();
+}