api: pause spam check when IPQualityScore credits run out

SD2 (14 §3): out of credits pauses IPQualityScore (one warning, one
event), the FTC list answers meanwhile, the dashboard shows a dismissable
spam_quota warning, and spamQuotaCheck resumes it every 6 hours once
credits are back. A newly saved key resumes it at once.

Follow-up: confirm with the owner's real key that the Credit Usage API
costs no credit (TODO in 14 §2.1).

apps/api/src/jobs.ts

@@ -31,6 +31,7 @@ import { logCallsToContrack } from './contacts/timeline.ts';
 import { checkDirectory } from './sso/directory.ts';
 import { checkOidcSessions } from './sso/oidcCheck.ts';
 import { logEvent } from './services/events.ts';
+import { spamQuotaCheck } from './spam/check.ts';
 import { deliverDue } from './services/webhooks.ts';
 import type { SendMail } from './services/mail.ts';
 import { getSetting, setSetting } from './services/settings.ts';
@@ -277,6 +278,7 @@ export function createJobs(d: JobDeps) {
   };
 
   return {
+    spamQuotaCheck: () => spamQuotaCheck(d.db),
     oidcCheck,
     directoryCheck,
     daily,
@@ -314,6 +316,7 @@ export function startJobs(d: JobDeps): () => void {
     run('contrackLog', j.contrackLog);
     if (minute % 5 === 0) run('oidcCheck', j.oidcCheck);
     if (minute % 60 === 0) run('directoryCheck', j.directoryCheck);
+    if (minute % 360 === 0) run('spamQuotaCheck', j.spamQuotaCheck);
     run('callTranscripts', async () => {
       await transcribeCalls({ db: d.db, dir: resolve(d.config.RECORDINGS_DIR), log: d.log });
     });

apps/api/src/routes/settings.ts

@@ -46,6 +46,7 @@ import { assertIssuer, oidcConfig, oidcError, testOidc, type OidcSettings } from
 import { ftcTest } from '../spam/ftc.ts';
 import { SpamError } from '../spam/http.ts';
 import { ipqsCredits } from '../spam/ipqs.ts';
+import { patchSpamState, resumeIpqs } from '../spam/check.ts';
 
 declare module 'fastify' {
   interface FastifyInstance {
@@ -350,6 +351,8 @@ export async function settingsRoutes(app: FastifyInstance) {
     }
     for (const [key, value] of Object.entries(values))
       await setSetting(db, key as SettingKey, value as never);
+    // A new IPQualityScore key may have credits: don't wait for spamQuotaCheck (14 §3).
+    if ((body.spam as SpamInput | undefined)?.ipqs?.key) await resumeIpqs(db, 'a new key');
     const changed = Object.keys(values).filter(
       (k) => JSON.stringify(before[k as keyof typeof before]) !== JSON.stringify(values[k]),
     );
@@ -556,6 +559,14 @@ export async function settingsRoutes(app: FastifyInstance) {
         message: s['system.directoryWarning'],
         link: '/admin/settings/sign-in',
       });
+    const st = s['spam.state'];
+    if (s.spam.enabled && s.spam.ipqs && st.ipqsPausedAt && !st.warningDismissedAt)
+      items.push({
+        code: 'spam_quota',
+        message: `IPQualityScore's lookups are used up. Spam checks ${s.spam.ftc ? 'use the FTC list' : 'are off'} until they come back.`,
+        link: '/admin/settings/spam',
+        dismissable: true,
+      });
     if (!s.smtp)
       items.push({
         code: 'smtp_missing',
@@ -564,4 +575,10 @@ export async function settingsRoutes(app: FastifyInstance) {
       });
     return { items };
   });
+
+  /** Hidden until the next time the credits run out (14 §3). */
+  app.post('/api/system/warnings/spam_quota/dismiss', async () => {
+    await patchSpamState(db, { warningDismissedAt: new Date().toISOString() });
+    return { ok: true };
+  });
 }

apps/api/src/spam/check.ts

@@ -2,16 +2,24 @@
 // The spam check (14 §5): the one way to ask what's known about a caller's number. Information
 // only: nothing here changes a call, and a failure only means there's no answer (and one warning
 // in the log). The contacts test (14 §1) is the caller's: it knows whose books to look in.
-import { and, eq, gt, sql } from 'drizzle-orm';
+import { and, eq, gt, isNull, sql, type SQL } from 'drizzle-orm';
 import type { FastifyBaseLogger } from 'fastify';
-import type { IpqsInfo, SpamCheck, SpamRisk, SpamSettings } from '@selfphoneng/shared';
+import {
+  SpamStateSchema,
+  type IpqsInfo,
+  type SpamCheck,
+  type SpamRisk,
+  type SpamSettings,
+  type SpamState,
+} from '@selfphoneng/shared';
 import type { Db } from '../db/client.ts';
-import { spamLookups } from '../db/schema.ts';
+import { settings, spamLookups } from '../db/schema.ts';
 import { decrypt } from '../crypto/secrets.ts';
+import { logEvent } from '../services/events.ts';
 import { getSetting } from '../services/settings.ts';
 import { ftcLookup } from './ftc.ts';
 import { SpamError } from './http.ts';
-import { ipqsLookup } from './ipqs.ts';
+import { ipqsCredits, ipqsLookup } from './ipqs.ts';
 
 export type SpamDeps = { db: Db; log: Pick<FastifyBaseLogger, 'warn'> };
 
@@ -44,7 +52,8 @@ export async function checkNumber(
       .from(spamLookups)
       .where(and(eq(spamLookups.number, number), gt(spamLookups.expiresAt, sql`now()`)));
     if (hit) return { risk: hit.risk as SpamRisk, provider: 'ipqs', info: hit.info as IpqsInfo };
-    if (!opts.cachedOnly) {
+    // Paused (14 §3): the cache still answers, IPQualityScore isn't asked.
+    if (!opts.cachedOnly && !(await getSetting(d.db, 'spam.state')).ipqsPausedAt) {
       let p = pending.get(number);
       if (!p) {
         p = askIpqs(d, s, s.ipqs.keyEnc, number).finally(() => pending.delete(number));
@@ -72,8 +81,7 @@ async function askIpqs(
   try {
     const a = await ipqsLookup(decrypt(keyEnc), number);
     if (a.kind === 'exhausted') {
-      // TODO(spec): SD2 pauses IPQualityScore here (14 §3); until then each check tries again.
-      d.log.warn({ provider: 'ipqs' }, 'spam check: IPQualityScore credits are used up');
+      await pauseIpqs(d, s);
       return null;
     }
     const days = a.risk === 'low' ? s.cache.cleanDays : s.cache.flaggedDays;
@@ -91,3 +99,67 @@ async function askIpqs(
     return null;
   }
 }
+
+/**
+ * Merges `patch` into spam.state in one statement, so the jobs and checks don't overwrite each
+ * other's fields. `onlyIf`: the condition the stored state must meet; false when it didn't.
+ */
+export async function patchSpamState(
+  db: Db,
+  patch: Partial<SpamState>,
+  onlyIf?: SQL,
+): Promise<boolean> {
+  await db
+    .insert(settings)
+    .values({ key: 'spam.state', value: SpamStateSchema.parse({}) })
+    .onConflictDoNothing();
+  const rows = await db
+    .update(settings)
+    .set({
+      value: sql`${settings.value} || ${JSON.stringify(patch)}::jsonb`,
+      updatedAt: new Date(),
+    })
+    .where(and(eq(settings.key, 'spam.state'), onlyIf))
+    .returning({ key: settings.key });
+  return rows.length > 0;
+}
+
+const paused = sql`${settings.value}->>'ipqsPausedAt'`;
+
+/** Out of credits (14 §3): paused, and the event, warning and log line once per time. */
+async function pauseIpqs(d: SpamDeps, s: SpamSettings) {
+  const first = await patchSpamState(
+    d.db,
+    { ipqsPausedAt: new Date().toISOString(), warningDismissedAt: null },
+    isNull(paused),
+  );
+  if (!first) return;
+  d.log.warn({ provider: 'ipqs' }, 'spam check: IPQualityScore credits are used up, paused');
+  await logEvent(d.db, {
+    severity: 'warning',
+    code: 'spam.quotaExhausted',
+    source: 'Spam check',
+    message: `IPQualityScore's lookups are used up. ${s.ftc ? 'Spam checks use the FTC list' : 'Spam checks are off'} until they come back.`,
+  });
+}
+
+/** Unpaused, by the job or a new key (14 §3). */
+export async function resumeIpqs(db: Db, why: string) {
+  const was = await patchSpamState(db, { ipqsPausedAt: null }, sql`${paused} is not null`);
+  if (was)
+    await logEvent(db, {
+      severity: 'info',
+      code: 'spam.quotaRestored',
+      source: 'Spam check',
+      message: `IPQualityScore lookups are back (${why}).`,
+    });
+}
+
+/** Job spamQuotaCheck (14 §3), every 6 hours: while paused, are credits back? A failure throws
+ *  (the job's log line) and the next run tries again. */
+export async function spamQuotaCheck(db: Db) {
+  const s = await getSetting(db, 'spam');
+  if (!s.ipqs || !(await getSetting(db, 'spam.state')).ipqsPausedAt) return;
+  const credits = await ipqsCredits(decrypt(s.ipqs.keyEnc));
+  if (credits > 0) await resumeIpqs(db, `${credits} credits left`);
+}

apps/api/test/spam.test.ts

@@ -1,12 +1,18 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-// SD1: the spam check (14): IPQualityScore first, then the local FTC copy, against a fake fetch.
-import { eq, sql } from 'drizzle-orm';
+// SD1, SD2: the spam check (14): IPQualityScore first, then the local FTC copy, against a fake fetch.
+import { eq, like, sql } from 'drizzle-orm';
 import type { LightMyRequestResponse } from 'fastify';
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
 import { SPAM_DEFAULTS, type SpamSettings } from '@selfphoneng/shared';
 import { encrypt } from '../src/crypto/secrets.ts';
-import { auditLog, ftcComplaints, spamLookups } from '../src/db/schema.ts';
-import { checkNumber } from '../src/spam/check.ts';
+import {
+  auditLog,
+  ftcComplaints,
+  settings as settingsTable,
+  spamLookups,
+  systemEvents,
+} from '../src/db/schema.ts';
+import { checkNumber, spamQuotaCheck } from '../src/spam/check.ts';
 import { getSetting, setSetting } from '../src/services/settings.ts';
 import { hasTestDb } from './db.ts';
 import { ADMIN, H, createTestApp } from './app.ts';
@@ -32,6 +38,7 @@ describe.skipIf(!hasTestDb)('spam check', () => {
   const urls: string[] = [];
   /** What the fake IPQualityScore does: answer, fail, or say the credits are used up. */
   let ipqs: 'ok' | 'down' | 'exhausted' | 'slow' = 'ok';
+  let credits = 912;
   const log = { warn: vi.fn() };
   const d = () => ({ db: t.db, log });
 
@@ -73,6 +80,7 @@ describe.skipIf(!hasTestDb)('spam check', () => {
     urls.length = 0;
     log.warn.mockClear();
     ipqs = 'ok';
+    credits = 912;
     vi.stubGlobal('fetch', async (url: string, init?: RequestInit) => {
       urls.push(url);
       const json = (j: unknown, status = 200) => new Response(JSON.stringify(j), { status });
@@ -83,7 +91,7 @@ describe.skipIf(!hasTestDb)('spam check', () => {
       const account = /\/api\/json\/account\/([^/]+)$/.exec(url);
       if (account)
         return account[1] === IPQS_KEY
-          ? json({ success: true, credits: 912, usage: 88 })
+          ? json({ success: true, credits, usage: 88 })
           : json({ success: false, message: `Invalid or unauthorized key: ${account[1]}` });
       const phone = /\/api\/json\/phone\/[^/]+\/(\d+)\?country\[\]=US$/.exec(url);
       if (!phone) return json({}, 404);
@@ -101,6 +109,8 @@ describe.skipIf(!hasTestDb)('spam check', () => {
   afterEach(async () => {
     vi.unstubAllGlobals();
     await t.db.delete(spamLookups);
+    await t.db.delete(settingsTable).where(eq(settingsTable.key, 'spam.state'));
+    await t.db.delete(systemEvents).where(like(systemEvents.code, 'spam.%'));
   });
 
   it('checks nothing while off, and only US numbers', async () => {
@@ -165,7 +175,7 @@ describe.skipIf(!hasTestDb)('spam check', () => {
 
   it('IPQualityScore failing: one warning without the key, nothing cached, the FTC answers', async () => {
     await settings({ ...keys.withIpqs, ...keys.withFtc });
-    for (const mode of ['down', 'slow', 'exhausted'] as const) {
+    for (const mode of ['down', 'slow'] as const) {
       ipqs = mode;
       expect(await checkNumber(d(), REPORTED)).toEqual({
         risk: 'high',
@@ -178,11 +188,9 @@ describe.skipIf(!hasTestDb)('spam check', () => {
         },
       });
     }
-    expect(log.warn).toHaveBeenCalledTimes(3);
     expect(log.warn.mock.calls.map((c) => c[0])).toEqual([
       { provider: 'ipqs', reason: 'HTTP 503' },
       { provider: 'ipqs', reason: 'no answer in 4 s' },
-      { provider: 'ipqs' },
     ]);
     expect(JSON.stringify(log.warn.mock.calls)).not.toContain(IPQS_KEY);
     expect(await t.db.select().from(spamLookups)).toEqual([]);
@@ -204,6 +212,89 @@ describe.skipIf(!hasTestDb)('spam check', () => {
     expect(await checkNumber(d(), REPORTED)).toBeNull();
   });
 
+  it('out of credits: paused once, the FTC answers, the warning, Dismiss, and back', async () => {
+    await settings({ ...keys.withIpqs, ...keys.withFtc });
+    const warning = async () =>
+      (await admin('GET', '/api/system/warnings'))
+        .json()
+        .items.find((w: { code: string }) => w.code === 'spam_quota');
+    const events = async () =>
+      (
+        await t.db
+          .select({ code: systemEvents.code })
+          .from(systemEvents)
+          .where(like(systemEvents.code, 'spam.%'))
+          .orderBy(systemEvents.createdAt)
+      ).map((e) => e.code);
+    ipqs = 'exhausted';
+    const both = await Promise.all([checkNumber(d(), REPORTED), checkNumber(d(), SHADY)]);
+    expect(both.map((a) => a?.provider)).toEqual(['ftc', 'ftc']);
+    expect((await getSetting(t.db, 'spam.state')).ipqsPausedAt).not.toBeNull();
+    expect(log.warn).toHaveBeenCalledTimes(1);
+    expect(await events()).toEqual(['spam.quotaExhausted']);
+    expect(await warning()).toEqual({
+      code: 'spam_quota',
+      message:
+        "IPQualityScore's lookups are used up. Spam checks use the FTC list until they come back.",
+      link: '/admin/settings/spam',
+      dismissable: true,
+    });
+
+    // Paused: IPQualityScore isn't asked and nothing is logged.
+    urls.length = 0;
+    expect((await checkNumber(d(), CLEAN))?.provider).toBe('ftc');
+    expect(urls).toEqual([]);
+    expect(log.warn).toHaveBeenCalledTimes(1);
+
+    expect((await admin('POST', '/api/system/warnings/spam_quota/dismiss')).statusCode).toBe(200);
+    expect(await warning()).toBeUndefined();
+
+    credits = 0;
+    await spamQuotaCheck(t.db);
+    expect((await getSetting(t.db, 'spam.state')).ipqsPausedAt).not.toBeNull();
+    credits = 5;
+    await spamQuotaCheck(t.db);
+    expect((await getSetting(t.db, 'spam.state')).ipqsPausedAt).toBeNull();
+    expect(await events()).toEqual(['spam.quotaExhausted', 'spam.quotaRestored']);
+    ipqs = 'ok';
+    expect((await checkNumber(d(), SPAMMER))?.provider).toBe('ipqs');
+
+    // Out again: the warning comes back though it was dismissed; without the FTC list, "off".
+    await settings(keys.withIpqs);
+    ipqs = 'exhausted';
+    expect(await checkNumber(d(), SHADY)).toBeNull();
+    expect((await warning())?.message).toBe(
+      "IPQualityScore's lookups are used up. Spam checks are off until they come back.",
+    );
+
+    // A new key doesn't wait for the job.
+    const r = await admin('PATCH', '/api/settings', {
+      spam: {
+        ...SPAM_DEFAULTS,
+        enabled: true,
+        ipqs: { key: IPQS_KEY },
+      },
+    });
+    expect(r.statusCode).toBe(200);
+    expect((await getSetting(t.db, 'spam.state')).ipqsPausedAt).toBeNull();
+    expect(await warning()).toBeUndefined();
+  });
+
+  it('spamQuotaCheck asks nothing unless paused, and a failure throws without the key', async () => {
+    await settings(keys.withIpqs);
+    await spamQuotaCheck(t.db);
+    expect(urls).toEqual([]);
+    await setSetting(t.db, 'spam.state', {
+      ...(await getSetting(t.db, 'spam.state')),
+      ipqsPausedAt: new Date().toISOString(),
+    });
+    await settings({ ipqs: { keyEnc: encrypt('wrong-key') } });
+    const err = await spamQuotaCheck(t.db).catch((e: unknown) => e as Error);
+    expect(err).toBeInstanceOf(Error);
+    expect(err!.message).toBe('Invalid or unauthorized key: ***');
+    expect((await getSetting(t.db, 'spam.state')).ipqsPausedAt).not.toBeNull();
+  });
+
   it('cachedOnly never asks IPQualityScore', async () => {
     await settings({ ...keys.withIpqs, ...keys.withFtc });
     expect((await checkNumber(d(), REPORTED, { cachedOnly: true }))?.provider).toBe('ftc');

apps/web/src/api/client.ts

@@ -256,6 +256,8 @@ export const settingsApi = {
       '/api/settings/emergency/test',
     ),
   warnings: () => api<{ items: Warning[] }>('GET', '/api/system/warnings').then((r) => r.items),
+  dismissWarning: (code: string) =>
+    api<{ ok: true }>('POST', `/api/system/warnings/${code}/dismiss`),
   testOidc: (v: OidcInput) => api<OidcTestResult>('POST', '/api/settings/auth/oidc/test', v),
   /** 12 §4.1 */
   testLdap: (v: LdapInput & { testLogin?: string; testPassword?: string }) =>

apps/web/src/pages/admin/DashboardPage.tsx

@@ -1,9 +1,9 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // Dashboard (05 §2). Warnings, telephony link, trunks, calls in progress (hang up / transfer)
 // queue tiles, parked calls and extensions online.
-import { useMutation, useQuery } from '@tanstack/react-query';
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { useEffect, useState, type ReactNode } from 'react';
-import type { ActiveCall, QueueLive } from '@selfphoneng/shared';
+import type { ActiveCall, QueueLive, Warning } from '@selfphoneng/shared';
 import { Link } from 'react-router';
 import { liveApi, settingsApi, type SpyMode } from '../../api/client.ts';
 import { ConfirmDialog } from '../../components/Dialog.tsx';
@@ -144,7 +144,12 @@ function Warnings({ trunkDown }: { trunkDown: boolean }) {
     queryFn: settingsApi.warnings,
     refetchInterval: 30_000,
   });
-  const items = [
+  const qc = useQueryClient();
+  const dismiss = useMutation({
+    mutationFn: settingsApi.dismissWarning,
+    onSuccess: () => qc.invalidateQueries({ queryKey: ['warnings'] }),
+  });
+  const items: Warning[] = [
     ...(trunkDown
       ? [
           {
@@ -168,6 +173,19 @@ function Warnings({ trunkDown }: { trunkDown: boolean }) {
           <Link to={w.link} className="font-medium underline">
             Fix
           </Link>
+          {w.dismissable && (
+            <>
+              {' '}
+              <button
+                type="button"
+                className="font-medium underline"
+                disabled={dismiss.isPending}
+                onClick={() => dismiss.mutate(w.code)}
+              >
+                Dismiss
+              </button>
+            </>
+          )}
         </li>
       ))}
     </ul>