jobs: public IP, trunk and disk watches, retention

apps/api/src/jobs.ts takes over the once-a-minute timer from index.ts
(no croner: every schedule in 04 §8 is whole minutes). Each job has a
running flag; errors are logged.

- Retention (at start and when the date changes, once AMI is up):
  recordings, voicemail past voicemail.retentionDays (VoicemailRemove, so
  Asterisk renumbers and updates MWI), expired sessions, webhook
  deliveries older than 30 days.
- publicIpWatch (every 5 min, Settings -> Network): a changed IP is saved,
  audited, applied (restart required) and emailed to admins.
- trunkWatch: a registering trunk rejected/unregistered for 5 minutes ->
  one email per outage.
- diskWatch (hourly): under 10% free -> one email a day.
- A failed daily backup emails admins too.

Settings gets a Network tab (public IP, Detect, re-check) and voicemail
retention on the Recording tab.

Task: M11-T4

apps/api/src/index.ts

@@ -17,10 +17,7 @@ import { watchToggles } from './services/toggles.ts';
 import { watchCdr } from './services/cdr.ts';
 import { watchQueueCalls } from './services/queuestats.ts';
 import { watchPinChanges } from './services/vmpin.ts';
-import { cleanupRecordings } from './routes/recordings.ts';
-import { getSetting } from './services/settings.ts';
-import { todayIn } from './generator/load.ts';
-import { backupDue, runBackup } from './services/backup.ts';
+import { startJobs } from './jobs.ts';
 
 let config;
 try {
@@ -108,42 +105,21 @@ ami.on('connected', () => {
 });
 ami.start();
 
-// Holidays roll over at midnight (03 §6.10): render again when the local date changes.
-// ponytail: a once-a-minute check instead of croner; the M11 jobs can take this over.
-let lastDay = '';
-const regenerateOnNewDay = async () => {
-  const today = todayIn(await getSetting(db, 'general.timezone'));
-  if (lastDay && today !== lastDay) await generator.apply({ trigger: 'nightly', force: true });
-  if (today !== lastDay) {
-    // Once a day (and at start): recordings past their retention (04 §8, 08).
-    const removed = await cleanupRecordings(
-      db,
-      resolve(config.RECORDINGS_DIR),
-      await getSetting(db, 'recordings.retentionDays'),
-    );
-    if (removed) app.log.info({ removed }, 'old recordings deleted');
-  }
-  lastDay = today;
-};
-/** The daily backup at backup.time (04 §8.1). A failure shows on the dashboard (backup_old). */
-let backupTriedDay = ''; // one try a day: a failing pg_dump shouldn't run every minute
-const backupIfDue = async () => {
-  const today = todayIn(await getSetting(db, 'general.timezone'));
-  if (backupTriedDay === today || !(await backupDue(config, db))) return;
-  backupTriedDay = today;
-  const b = await runBackup(config, db);
-  app.log.info({ backup: b.name, size: b.size }, 'backup made');
-};
-void regenerateOnNewDay();
-const dayTimer = setInterval(() => {
-  void regenerateOnNewDay().catch((err) => app.log.error({ err }, 'nightly regenerate failed'));
-  void backupIfDue().catch((err) => app.log.error({ err }, 'backup failed'));
-}, 60_000);
+const stopJobs = startJobs({
+  config,
+  db,
+  ami,
+  live,
+  generator,
+  voicemail,
+  send: sendMail,
+  log: app.log,
+});
 
 for (const sig of ['SIGTERM', 'SIGINT'] as const) {
   process.once(sig, () => {
     generator.cancelPending();
-    clearInterval(dayTimer);
+    stopJobs();
     history.stop();
     ami.stop();
     void app

apps/api/src/jobs.ts

@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Background jobs (04 §8) on one once-a-minute timer.
+// ponytail: setInterval instead of croner: every schedule in 04 §8 is a whole number of minutes.
+import { statfs } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { and, eq, isNull, lt, sql } from 'drizzle-orm';
+import { z } from 'zod';
+import type { Config } from './config.ts';
+import type { Db } from './db/client.ts';
+import {
+  extensions,
+  sessions,
+  trunks,
+  users,
+  voicemailMessages,
+  webhookDeliveries,
+} from './db/schema.ts';
+import type { Generator } from './generator/index.ts';
+import { todayIn } from './generator/load.ts';
+import type { LiveState } from './live/state.ts';
+import { cleanupRecordings } from './routes/recordings.ts';
+import { audit } from './services/audit.ts';
+import { backupDue, runBackup } from './services/backup.ts';
+import type { SendMail } from './services/mail.ts';
+import { getSetting, setSetting } from './services/settings.ts';
+
+export type JobDeps = {
+  config: Config;
+  db: Db;
+  ami: { isConnected(): boolean; action(f: Record<string, string>): Promise<unknown> };
+  live: Pick<LiveState, 'snapshot' | 'amiConnected'>;
+  generator: Pick<Generator, 'apply'>;
+  voicemail: { scan(ext: string): Promise<void> };
+  send: SendMail;
+  log: { info(o: object, m: string): void; error(o: object, m: string): void };
+  now?: () => Date;
+  statfs?: typeof statfs;
+};
+
+/** The server's public IPv4 as an outside service sees it, or null. */
+export async function detectPublicIp(): Promise<string | null> {
+  for (const url of ['https://api.ipify.org', 'https://ifconfig.me/ip']) {
+    const ip = await fetch(url, { signal: AbortSignal.timeout(5000) })
+      .then((r) => (r.ok ? r.text() : ''))
+      .catch(() => '');
+    if (z.ipv4().safeParse(ip.trim()).success) return ip.trim();
+  }
+  return null;
+}
+
+/** Email every active admin (08 §3.2). Email not set up or failing is logged, not thrown. */
+export async function mailAdmins(d: JobDeps, subject: string, text: string) {
+  const to = (
+    await d.db
+      .select({ email: users.email })
+      .from(users)
+      .where(and(eq(users.role, 'admin'), eq(users.isActive, true)))
+  ).map((r) => r.email);
+  if (to.length)
+    await d.send({ to, subject, text }).catch((err: unknown) => d.log.error({ err }, subject));
+}
+
+/** Recordings, voicemail, expired sessions and old webhook deliveries (04 §8 retentionCleanup). */
+export async function retentionCleanup(d: JobDeps) {
+  const recordings = await cleanupRecordings(
+    d.db,
+    resolve(d.config.RECORDINGS_DIR),
+    await getSetting(d.db, 'recordings.retentionDays'),
+  );
+  let voicemail = 0;
+  const vmDays = await getSetting(d.db, 'voicemail.retentionDays');
+  // Asterisk removes the files and renumbers the folder (the caller waits for AMI).
+  if (vmDays > 0) {
+    const old = await d.db
+      .select({
+        ext: extensions.number,
+        folder: voicemailMessages.folder,
+        msgId: voicemailMessages.msgId,
+      })
+      .from(voicemailMessages)
+      .innerJoin(extensions, eq(extensions.id, voicemailMessages.extensionId))
+      .where(
+        and(
+          isNull(voicemailMessages.deletedAt),
+          lt(voicemailMessages.receivedAt, sql`now() - make_interval(days => ${vmDays})`),
+        ),
+      );
+    for (const m of old)
+      await d.ami.action({
+        Action: 'VoicemailRemove',
+        Context: 'default',
+        Mailbox: m.ext,
+        Folder: m.folder,
+        ID: m.msgId,
+      });
+    for (const ext of new Set(old.map((m) => m.ext))) await d.voicemail.scan(ext);
+    voicemail = old.length;
+  }
+  await d.db.delete(sessions).where(lt(sessions.expiresAt, sql`now()`));
+  await d.db
+    .delete(webhookDeliveries)
+    .where(lt(webhookDeliveries.createdAt, sql`now() - interval '30 days'`));
+  if (recordings || voicemail) d.log.info({ recordings, voicemail }, 'old messages deleted');
+}
+
+/** The jobs, each keeping its own state (tests call them directly). */
+export function createJobs(d: JobDeps) {
+  const now = d.now ?? (() => new Date());
+  const tz = () => getSetting(d.db, 'general.timezone');
+
+  // Holidays roll over at midnight (03 §6.10): render again when the local date changes.
+  // Retention runs then too (and at start), not at 02:00.
+  let lastDay = '';
+  let cleanedDay = '';
+  const daily = async () => {
+    const today = todayIn(await tz(), now());
+    if (lastDay && today !== lastDay) await d.generator.apply({ trigger: 'nightly', force: true });
+    lastDay = today;
+    // Voicemail goes through Asterisk: at start, wait the few seconds until AMI is connected.
+    if (cleanedDay !== today && d.ami.isConnected()) {
+      await retentionCleanup(d);
+      cleanedDay = today;
+    }
+  };
+
+  let backupTriedDay = ''; // one try a day: a failing pg_dump shouldn't run every minute
+  const backup = async () => {
+    const today = todayIn(await tz(), now());
+    if (backupTriedDay === today || !(await backupDue(d.config, d.db, now()))) return;
+    backupTriedDay = today;
+    try {
+      const b = await runBackup(d.config, d.db);
+      d.log.info({ backup: b.name, size: b.size }, 'backup made');
+    } catch (err) {
+      await mailAdmins(d, 'Backup failed', `Today's backup failed: ${(err as Error).message}`);
+      throw err;
+    }
+  };
+
+  const publicIpWatch = async () => {
+    if (!(await getSetting(d.db, 'network.publicIpWatch'))) return;
+    const ip = await detectPublicIp();
+    const old = await getSetting(d.db, 'network.publicIp');
+    if (!ip || ip === old) return;
+    await setSetting(d.db, 'network.publicIp', ip);
+    await audit(d.db, {
+      action: 'settings',
+      summary: `Public IP changed from ${old ?? 'none'} to ${ip}`,
+      diff: { before: old, after: ip },
+    });
+    await d.generator.apply({ trigger: 'publicIp', force: true }); // sets restartRequired
+    await mailAdmins(
+      d,
+      `Public IP changed to ${ip}`,
+      `The phone system's public IP changed from ${old ?? 'none'} to ${ip}.\n` +
+        'Restart telephony from the dashboard so calls use the new address (it drops calls in progress).\n' +
+        'Phones and the provider may need the new IP if they were set up with the old one.',
+    );
+  };
+
+  // A registering trunk that stays down for 5 minutes: one email per outage.
+  const downSince = new Map<string, number>();
+  const mailed = new Set<string>();
+  const trunkWatch = async () => {
+    if (!d.live.amiConnected) return; // Asterisk itself being down shows elsewhere
+    const state = new Map((d.live.snapshot(true).trunks ?? []).map((t) => [t.trunkId, t]));
+    const rows = await d.db
+      .select({ id: trunks.id, name: trunks.name })
+      .from(trunks)
+      .where(eq(trunks.enabled, true));
+    for (const t of rows) {
+      const reg = state.get(t.id)?.registration;
+      // ponytail: IP-auth trunks don't register (always 'unknown'), so they're never "down" here.
+      if (reg !== 'rejected' && reg !== 'unregistered') {
+        downSince.delete(t.id);
+        mailed.delete(t.id);
+        continue;
+      }
+      const since = downSince.get(t.id) ?? now().getTime();
+      downSince.set(t.id, since);
+      if (now().getTime() - since < 5 * 60_000 || mailed.has(t.id)) continue;
+      mailed.add(t.id);
+      await mailAdmins(
+        d,
+        `Trunk ${t.name} is down`,
+        `Trunk ${t.name} has not been registered with the provider since ` +
+          `${new Date(since).toISOString()} (${reg}). Calls through it fail.\n` +
+          'Check Trunks → Test, the account password, and the provider status.',
+      );
+    }
+  };
+
+  let diskMailedDay = '';
+  const diskWatch = async () => {
+    const fs = await (d.statfs ?? statfs)(resolve(d.config.RECORDINGS_DIR));
+    const free = fs.bavail / fs.blocks;
+    const today = todayIn(await tz(), now());
+    if (free >= 0.1 || diskMailedDay === today) return;
+    diskMailedDay = today;
+    await mailAdmins(
+      d,
+      `Disk almost full (${Math.round(free * 100)}% free)`,
+      `Only ${Math.round((fs.bavail * fs.bsize) / 1e9)} GB (${Math.round(free * 100)}%) is free ` +
+        'on the phone system. Recordings, voicemail and backups stop being saved when it fills.\n' +
+        'Lower the recording retention (Settings → Recording) or free up space.',
+    );
+  };
+
+  return { daily, backup, publicIpWatch, trunkWatch, diskWatch };
+}
+
+/** Starts the jobs; returns a stop function. */
+export function startJobs(d: JobDeps): () => void {
+  const j = createJobs(d);
+  // Every job: never two runs of itself at once (04 §8), errors logged.
+  const running = new Set<string>();
+  const run = (name: string, fn: () => Promise<void>) => {
+    if (running.has(name)) return;
+    running.add(name);
+    void fn()
+      .catch((err: unknown) => d.log.error({ err }, `job ${name} failed`))
+      .finally(() => running.delete(name));
+  };
+  let minute = 0;
+  const tick = () => {
+    run('daily', j.daily);
+    run('backup', j.backup);
+    run('trunkWatch', j.trunkWatch);
+    if (minute % 5 === 0) run('publicIpWatch', j.publicIpWatch);
+    if (minute % 60 === 0) run('diskWatch', j.diskWatch);
+    minute++;
+  };
+  tick();
+  const timer = setInterval(tick, 60_000);
+  return () => clearInterval(timer);
+}

apps/api/src/routes/settings.ts

@@ -22,6 +22,7 @@ import { getAllSettings, getSetting, setSetting } from '../services/settings.ts'
 import { notifyEmergency } from '../services/emergency.ts';
 import { readTlsCert } from '../services/tls.ts';
 import { backupMissed } from '../services/backup.ts';
+import { detectPublicIp } from '../jobs.ts';
 import type { SendMail } from '../services/mail.ts';
 
 declare module 'fastify' {
@@ -179,12 +180,8 @@ export async function settingsRoutes(app: FastifyInstance) {
 
   /** The server's public IPv4, as an outside service sees it (only sent when asked). */
   app.post('/api/settings/network/detect-public-ip', async () => {
-    for (const url of ['https://api.ipify.org', 'https://ifconfig.me/ip']) {
-      const ip = await fetch(url, { signal: AbortSignal.timeout(5000) })
-        .then((r) => (r.ok ? r.text() : ''))
-        .catch(() => '');
-      if (z.ipv4().safeParse(ip.trim()).success) return { ip: ip.trim() };
-    }
+    const ip = await detectPublicIp();
+    if (ip) return { ip };
     throw new HttpError(502, 'unavailable', "Couldn't detect the public IP. Enter it by hand.");
   });
 

apps/api/test/jobs.test.ts

@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// M11-T4: background jobs (04 §8): retention, trunk/disk/public IP watches.
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import type { LightMyRequestResponse } from 'fastify';
+import { eq } from 'drizzle-orm';
+import type { TrunkLive } from '@selfphoneng/shared';
+import { createJobs, retentionCleanup, type JobDeps } from '../src/jobs.ts';
+import { extensions, sessions, voicemailMessages } from '../src/db/schema.ts';
+import { getSetting, setSetting } from '../src/services/settings.ts';
+import { hasTestDb } from './db.ts';
+import { ADMIN, H, createTestApp } from './app.ts';
+
+describe.skipIf(!hasTestDb)('background jobs', () => {
+  let t: Awaited<ReturnType<typeof createTestApp>>;
+  let req: (m: string, url: string, payload?: unknown) => Promise<LightMyRequestResponse>;
+  let trunkId: string;
+  let clock = Date.parse('2026-10-19T12:00:00Z');
+  let reg: TrunkLive['registration'] = 'registered';
+  const scanned: string[] = [];
+  const deps = (): JobDeps => ({
+    config: t.app.config,
+    db: t.db,
+    ami: t.ami,
+    live: {
+      amiConnected: true,
+      snapshot: () => ({
+        extensions: [],
+        queues: [],
+        parked: [],
+        ami: { connected: true },
+        trunks: [{ trunkId, registration: reg, reachable: true, lastChange: '' }],
+      }),
+    },
+    generator: t.app.generator,
+    voicemail: { scan: async (ext) => void scanned.push(ext) },
+    send: t.app.sendMail,
+    log: t.app.log,
+    now: () => new Date(clock),
+  });
+
+  beforeAll(async () => {
+    t = await createTestApp();
+    await t.clearMustChange(ADMIN.email);
+    const cookie = await t.login(ADMIN.email, ADMIN.password);
+    req = (method, url, payload) =>
+      t.app.inject({
+        method: method as 'GET',
+        url,
+        payload: payload as object,
+        headers: { ...H, cookie },
+      });
+    await req('POST', '/api/extensions', { number: '101', name: 'Dan' });
+    const r = await req('POST', '/api/trunks', {
+      name: 'voip.ms',
+      serverHost: '127.0.0.1',
+      username: '123456_pbx',
+      password: 'Sub-account pw!',
+    });
+    trunkId = r.json().id;
+  });
+  afterAll(() => t.close());
+  afterEach(() => vi.unstubAllGlobals());
+
+  it('retention removes old voicemail through Asterisk, and expired sessions', async () => {
+    const [ext] = await t.db.select().from(extensions).where(eq(extensions.number, '101'));
+    const msg = (msgId: string, days: number) => ({
+      extensionId: ext!.id,
+      msgId,
+      folder: 'INBOX',
+      durationS: 3,
+      receivedAt: new Date(Date.now() - days * 86_400_000),
+      filePath: `default/101/INBOX/${msgId}.wav`,
+    });
+    await t.db.insert(voicemailMessages).values([msg('old-1', 40), msg('new-1', 2)]);
+    const [u] = await t.db.select({ id: sessions.userId }).from(sessions);
+    await t.db.insert(sessions).values({
+      id: 'expired-session-hash',
+      userId: u!.id,
+      csrfToken: 'x',
+      expiresAt: new Date(Date.now() - 1000),
+    } as typeof sessions.$inferInsert);
+
+    await setSetting(t.db, 'voicemail.retentionDays', 0); // 0 keeps voicemail
+    t.ami.actions.length = 0;
+    await retentionCleanup(deps());
+    expect(t.ami.actions).toEqual([]);
+    expect(
+      await t.db.select().from(sessions).where(eq(sessions.id, 'expired-session-hash')),
+    ).toEqual([]);
+
+    await setSetting(t.db, 'voicemail.retentionDays', 30);
+    await retentionCleanup(deps());
+    expect(t.ami.actions).toEqual([
+      {
+        Action: 'VoicemailRemove',
+        Context: 'default',
+        Mailbox: '101',
+        Folder: 'INBOX',
+        ID: 'old-1',
+      },
+    ]);
+    expect(scanned).toEqual(['101']);
+
+    // The daily job waits for Asterisk (at start AMI connects a few seconds after the first run).
+    await t.db.update(voicemailMessages).set({ deletedAt: null }); // fake scan: still there
+    const j = createJobs(deps());
+    t.ami.connected = false;
+    t.ami.actions.length = 0;
+    await j.daily();
+    expect(t.ami.actions).toEqual([]);
+    t.ami.connected = true;
+    await j.daily();
+    await j.daily(); // once a day
+    expect(t.ami.actions.map((a) => a.ID)).toEqual(['old-1']);
+  });
+
+  it('trunkWatch emails admins once when a trunk is down for 5 minutes', async () => {
+    const j = createJobs(deps());
+    t.mails.length = 0;
+    reg = 'rejected';
+    await j.trunkWatch();
+    clock += 4 * 60_000;
+    await j.trunkWatch();
+    expect(t.mails).toEqual([]);
+    clock += 60_000;
+    await j.trunkWatch();
+    clock += 60_000;
+    await j.trunkWatch();
+    expect(t.mails.map((m) => [m.to, m.subject])).toEqual([
+      [[ADMIN.email], 'Trunk voip.ms is down'],
+    ]);
+    // Back up, then down again: a new outage gets its own email.
+    reg = 'registered';
+    await j.trunkWatch();
+    reg = 'unregistered';
+    await j.trunkWatch();
+    clock += 5 * 60_000;
+    await j.trunkWatch();
+    expect(t.mails).toHaveLength(2);
+    // IP-auth trunks never register: never "down".
+    reg = 'unknown';
+    const k = createJobs(deps());
+    await k.trunkWatch();
+    clock += 10 * 60_000;
+    await k.trunkWatch();
+    expect(t.mails).toHaveLength(2);
+  });
+
+  it('publicIpWatch saves a changed IP, asks for a restart and emails admins', async () => {
+    const j = createJobs(deps());
+    vi.stubGlobal('fetch', async () => new Response('203.0.113.9\n'));
+    t.mails.length = 0;
+    await setSetting(t.db, 'network.publicIp', '203.0.113.5');
+    await j.publicIpWatch(); // off by default
+    expect(await getSetting(t.db, 'network.publicIp')).toBe('203.0.113.5');
+
+    await setSetting(t.db, 'network.publicIpWatch', true);
+    await j.publicIpWatch();
+    expect(await getSetting(t.db, 'network.publicIp')).toBe('203.0.113.9');
+    expect(await getSetting(t.db, 'system.restartRequired')).toBe(true);
+    expect(t.mails.map((m) => m.subject)).toEqual(['Public IP changed to 203.0.113.9']);
+
+    await j.publicIpWatch(); // unchanged: nothing
+    vi.stubGlobal('fetch', async () => new Response('<html>rate limited</html>'));
+    await j.publicIpWatch(); // garbage answer: keep the old IP
+    expect(await getSetting(t.db, 'network.publicIp')).toBe('203.0.113.9');
+    expect(t.mails).toHaveLength(1);
+  });
+
+  it('diskWatch emails admins once a day under 10% free', async () => {
+    let bavail = 50;
+    const fake = (async () => ({
+      bavail,
+      blocks: 100,
+      bsize: 1e9,
+    })) as unknown as JobDeps['statfs'];
+    const j = createJobs({ ...deps(), statfs: fake });
+    t.mails.length = 0;
+    await j.diskWatch();
+    expect(t.mails).toEqual([]);
+    bavail = 6;
+    await j.diskWatch();
+    await j.diskWatch();
+    expect(t.mails.map((m) => [m.subject, m.text.split('.')[0]])).toEqual([
+      ['Disk almost full (6% free)', 'Only 6 GB (6%) is free on the phone system'],
+    ]);
+    clock += 86_400_000; // next day: again
+    await j.diskWatch();
+    expect(t.mails).toHaveLength(2);
+  });
+
+  it('a mail that fails to send is logged, not thrown', async () => {
+    t.mail.fail = 'smtp down';
+    reg = 'rejected';
+    const j = createJobs(deps());
+    await j.trunkWatch();
+    clock += 6 * 60_000;
+    await expect(j.trunkWatch()).resolves.toBeUndefined();
+    t.mail.fail = null;
+  });
+});