Feat/spam #5

Merged
iamdoubz merged 7 commits from feat/spam into main 2026-10-27 18:35:05 -05:00
11 changed files with 185 additions and 14 deletions
Showing only changes of commit 6b0c5aa827 - Show all commits

No files matched your search

+18 -5
View File
@@ -10,10 +10,12 @@ import {
type CallDto,
type ListResponse,
type MyCallDto,
type SpamCheck,
} from '@selfphoneng/shared';
import { requireAdmin, requireAuth } from '../auth/guards.ts';
import type { Db } from '../db/client.ts';
import { callExtensions, calls, cdr, dids, extensions } from '../db/schema.ts';
import { contactNames } from '../contacts/book.ts';
import { badRequest, notFound } from '../errors.ts';
import { getSetting } from '../services/settings.ts';
@@ -65,7 +67,8 @@ async function conditions(db: Db, f: Filter): Promise<SQL[]> {
return where;
}
async function page(db: Db, where: SQL[], limit: number) {
/** `viewer`: whose books hide the spam check for a number they name (14 §6). */
async function page(db: Db, where: SQL[], limit: number, viewer: string) {
const rows = await db
.select({ c: calls, did: dids, a: answerer })
.from(calls)
@@ -76,6 +79,12 @@ async function page(db: Db, where: SQL[], limit: number) {
.limit(limit + 1);
const more = rows.length > limit;
const items = rows.slice(0, limit);
const known = await contactNames(
db,
viewer,
items.filter(({ c }) => c.spamRisk).map(({ c }) => c.fromNumber),
{ wait: false },
).catch(() => null);
return {
items: items.map(({ c, did, a }): CallDto => ({
id: c.id,
@@ -91,6 +100,10 @@ async function page(db: Db, where: SQL[], limit: number) {
answeredBy: a ? { id: a.id, number: a.number, name: a.name } : null,
talkSeconds: c.talkSeconds,
recordingId: c.recordingId,
spam:
c.spamRisk && known && !known.has(c.fromNumber ?? '')
? ({ risk: c.spamRisk, ...(c.spamInfo as object) } as SpamCheck)
: null,
})),
nextCursor: more ? encodeCursor(items.at(-1)!.c) : null,
};
@@ -111,7 +124,7 @@ export async function callRoutes(app: FastifyInstance) {
{ preHandler: requireAdmin },
async (req): Promise<ListResponse<CallDto>> => {
const f = CallFilterSchema.parse(req.query);
return page(db, await conditions(db, f), f.limit);
return page(db, await conditions(db, f), f.limit, req.user!.id);
},
);
@@ -123,7 +136,7 @@ export async function callRoutes(app: FastifyInstance) {
'started,direction,status,from,from_name,to,phone_number,answered_by,talk_seconds',
];
// ponytail: one query capped at 50,000 rows; stream with a cursor if exports get bigger.
const { items } = await page(db, where, 50_000);
const { items } = await page(db, where, 50_000, req.user!.id);
const fmt = new Intl.DateTimeFormat('sv-SE', {
timeZone: tz,
dateStyle: 'short',
@@ -156,7 +169,7 @@ export async function callRoutes(app: FastifyInstance) {
app.get('/api/calls/:id', { preHandler: requireAdmin }, async (req): Promise<CallDetailDto> => {
const { id } = z.object({ id: z.uuid() }).parse(req.params);
const { items } = await page(db, [eq(calls.id, id)], 1);
const { items } = await page(db, [eq(calls.id, id)], 1, req.user!.id);
const [c] = await db.select({ linkedid: calls.linkedid }).from(calls).where(eq(calls.id, id));
if (!items[0] || !c) throw notFound('Call not found');
const legs = await db
@@ -189,7 +202,7 @@ export async function callRoutes(app: FastifyInstance) {
.where(eq(extensions.userId, req.user!.id));
if (!ext) throw notFound("You don't have an extension yet. Ask your admin.");
const where = await conditions(db, { ...f, extensionId: ext.id });
const result = await page(db, where, f.limit);
const result = await page(db, where, f.limit, req.user!.id);
const ids = result.items.map((c) => c.id);
const roles = ids.length
? await db
+14 -4
View File
@@ -9,6 +9,7 @@ import { join } from 'node:path';
import { and, asc, eq, inArray, isNull, lt } from 'drizzle-orm';
import { field, type AmiMessage } from '../ami/events.ts';
import type { Db } from '../db/client.ts';
import { checkNumber, usNumber, type SpamDeps } from '../spam/check.ts';
import { callExtensions, calls, cdr, dids, extensions, recordings } from '../db/schema.ts';
import { durationMs } from '../media/audio.ts';
@@ -160,7 +161,7 @@ async function linkRecording(db: Db | Tx, dir: string, s: CallSummary, linkedid:
export async function aggregateCall(
db: Db,
linkedid: string,
opts: { recordingsDir?: string } = {},
opts: { recordingsDir?: string; log?: SpamDeps['log'] } = {},
): Promise<void> {
const rows = await db
.select()
@@ -180,6 +181,13 @@ export async function aggregateCall(
const [d] = s.did
? await db.select({ id: dids.id }).from(dids).where(eq(dids.number, s.did))
: [];
// 14 §6: what the spam check knows now (the cache, the local FTC copy), kept with the call.
const spam =
s.direction === 'inbound' && s.fromNumber
? await checkNumber({ db, log: opts.log ?? { warn() {} } }, usNumber(s.fromNumber), {
cachedOnly: true,
})
: null;
await db.transaction(async (tx) => {
await tx.delete(calls).where(eq(calls.linkedid, linkedid));
const [call] = await tx
@@ -197,6 +205,8 @@ export async function aggregateCall(
didId: d?.id ?? null,
answeredByExtensionId: idOf(s.answeredBy),
talkSeconds: s.talkSeconds,
spamRisk: spam?.risk ?? null,
spamInfo: spam && { provider: spam.provider, info: spam.info },
recordingId: opts.recordingsDir
? await linkRecording(tx, opts.recordingsDir, s, linkedid)
: null,
@@ -218,7 +228,7 @@ export async function aggregateCall(
export type CdrDeps = {
db: Db;
log: { error(o: object, msg: string): void };
log: { error(o: object, msg: string): void; warn(o: object, msg: string): void };
/** How long after a call's last Cdr event to aggregate it (04 §5.4: 5 s). */
settleMs?: number;
/** Where MixMonitor writes (RECORDINGS_DIR); recordings are linked to their calls. */
@@ -233,7 +243,7 @@ export function watchCdr(d: CdrDeps, ami: Pick<EventEmitter, 'on'>) {
linkedid,
setTimeout(() => {
timers.delete(linkedid);
void aggregateCall(d.db, linkedid, { recordingsDir: d.recordingsDir }).catch(
void aggregateCall(d.db, linkedid, { recordingsDir: d.recordingsDir, log: d.log }).catch(
(err: unknown) => d.log.error({ err, linkedid }, 'call aggregation failed'),
);
}, d.settleMs ?? 5000),
@@ -257,7 +267,7 @@ export function watchCdr(d: CdrDeps, ami: Pick<EventEmitter, 'on'>) {
.leftJoin(calls, eq(calls.linkedid, cdr.linkedid))
.where(and(isNull(calls.id), lt(cdr.createdAt, new Date(Date.now() - 60_000))));
for (const o of orphans)
await aggregateCall(d.db, o.linkedid, { recordingsDir: d.recordingsDir });
await aggregateCall(d.db, o.linkedid, { recordingsDir: d.recordingsDir, log: d.log });
},
stop: () => timers.forEach(clearTimeout),
};
+1 -2
View File
@@ -3,7 +3,6 @@
// only: nothing here changes a call, and a failure only means there's no answer (and one warning
// in the log). The contacts test (14 §1) is the caller's: it knows whose books to look in.
import { and, eq, gt, isNull, sql, type SQL } from 'drizzle-orm';
import type { FastifyBaseLogger } from 'fastify';
import {
SpamStateSchema,
normalizeNumber,
@@ -23,7 +22,7 @@ import { ftcLookup } from './ftc.ts';
import { SpamError } from './http.ts';
import { ipqsCredits, ipqsLookup } from './ipqs.ts';
export type SpamDeps = { db: Db; log: Pick<FastifyBaseLogger, 'warn'> };
export type SpamDeps = { db: Db; log: { warn(o: object, msg: string): void } };
/** US numbers only (owner, 2026-10-27): 1NXXNXXXXXX after normalizeNumber (00 §8). */
export const isCheckable = (n: string) => /^1[2-9]\d{2}[2-9]\d{6}$/.test(n);
+41 -1
View File
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// SD1, SD2, SD4: the spam check (14): IPQualityScore first, then the local FTC copy, against a fake fetch.
// SD1, SD2, SD4, SD5: the spam check (14): IPQualityScore first, then the local FTC copy, against a fake fetch.
import { readFileSync } from 'node:fs';
import { eq, like, sql } from 'drizzle-orm';
import type { LightMyRequestResponse } from 'fastify';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
@@ -7,6 +8,7 @@ import { SPAM_DEFAULTS, type SpamSettings } from '@selfphoneng/shared';
import { encrypt } from '../src/crypto/secrets.ts';
import {
auditLog,
cdr,
ftcComplaints,
settings as settingsTable,
spamLookups,
@@ -14,7 +16,9 @@ import {
users,
} from '../src/db/schema.ts';
import type { ActiveCall } from '@selfphoneng/shared';
import type { AmiMessage } from '../src/ami/events.ts';
import { callerNameToken } from '../src/contacts/callername.ts';
import { aggregateCall, toCdrRow } from '../src/services/cdr.ts';
import { checkNumber, spamQuotaCheck } from '../src/spam/check.ts';
import { getSetting, setSetting } from '../src/services/settings.ts';
import { hasTestDb } from './db.ts';
@@ -369,6 +373,42 @@ describe.skipIf(!hasTestDb)('spam check', () => {
await settings({ ...keys.withFtc, deskPhoneLabels: { high: 'JUNK', medium: 'Maybe junk' } });
expect(await deskName(REPORTED)).toBe('JUNK');
});
it('call history keeps what the check knew, after the cache expires; contacts first', async () => {
const fixture = readFileSync(
new URL('./fixtures/cdr/inbound-user.json', import.meta.url),
'utf8',
);
const callFrom = async (number: string) => {
const events = (JSON.parse(fixture.replaceAll('13125559999', number)) as AmiMessage[]).map(
(e) => ({ ...e, LinkedID: `${number}-${String(e.LinkedID)}` }),
);
for (const e of events) await t.db.insert(cdr).values(toCdrRow(e));
await aggregateCall(t.db, String(events[0]!.LinkedID));
};
await checkNumber(d(), SPAMMER);
urls.length = 0;
for (const n of [SPAMMER, REPORTED, SHADY, CLEAN]) await callFrom(n);
expect(urls).toEqual([]); // cachedOnly: CLEAN isn't asked
await t.db.delete(spamLookups);
const spamOf = async (n: string) =>
(await admin('GET', '/api/calls?direction=inbound'))
.json()
.items.find((c: { fromNumber: string }) => c.fromNumber === n)?.spam;
expect(await spamOf(SPAMMER)).toMatchObject({
risk: 'high',
provider: 'ipqs',
info: { spammer: true },
});
expect(await spamOf(REPORTED)).toMatchObject({
risk: 'high',
provider: 'ftc',
info: { complaints: 3 },
});
expect(await spamOf(SHADY)).toBeNull(); // the admin's contact
// Not in the cache, never reported: low (the page shows nothing for it).
expect(await spamOf(CLEAN)).toMatchObject({ risk: 'low', provider: 'ftc' });
});
});
describe('Settings → Spam check', () => {
+2
View File
@@ -11,6 +11,7 @@ import {
type CallFilter,
} from '@selfphoneng/shared';
import { callsApi, extensionsApi, recordingsApi } from '../../api/client.ts';
import { SpamNote } from '../../components/SpamNote.tsx';
import { Alert, Button, Field, inputClass } from '../../components/ui.tsx';
export const fmtNumber = (n: string | null) =>
@@ -170,6 +171,7 @@ export function CallsPage() {
{c.answeredBy && (
<span className="text-zinc-500"> · answered by {c.answeredBy.name}</span>
)}
<SpamNote check={c.spam} />
</span>
<StatusBadge status={c.status} />
<span className="w-12 text-right tabular-nums">{fmtDuration(c.talkSeconds)}</span>
+2
View File
@@ -9,6 +9,7 @@ import { Alert, Button } from '../../components/ui.tsx';
import { softphone } from '../../softphone/SoftphoneEngine.ts';
import { DIRECTION_TEXT, StatusBadge, fmtDuration, fmtNumber } from '../admin/CallsPage.tsx';
import { AddToContacts, useContactName } from './ContactsPage.tsx';
import { SpamNote } from '../../components/SpamNote.tsx';
/** The other side of the call, from where I sit. */
export const otherParty = (c: MyCallDto) =>
@@ -84,6 +85,7 @@ export function MyCallsPage() {
{!known && other.number && c.direction !== 'internal' && (
<AddToContacts number={other.number} name={other.name} />
)}
{c.myRole === 'in' && !known && <SpamNote check={c.spam} />}
</span>
{c.recordingId && (
<audio
+1
View File
@@ -13,6 +13,7 @@ keeping the system safe, legal and up to date.
8. [Local AI](ai.md): voicemail transcripts
9. [Contacts](contacts.md): the builtin phonebook, or Contrack
10. [Sign-in](sign-in.md): passwords, and OpenID Connect (Keycloak and others)
11. [Spam check](spam-check.md): IPQualityScore and the FTC complaint list
---
+87
View File
@@ -0,0 +1,87 @@
# Spam check
When an outside caller from the US isn't anyone's contact, SelfPhoneNG can look the number up
and show what's known about it while it rings:
- **Browser phone:** a red **Likely spam** or amber **Possible spam** badge under the caller's
name, with a line such as "Fraud score 92 · spammer · VOIP · Austin, TX" or "Reported to the
FTC 4 times (3 robocalls), last on Oct 20: Warranties".
- **Desk phones:** the caller's name shows as **SPAM LIKELY** or **Spam risk** (you can change
these), if the answer comes in before the phone starts ringing (about a second).
- **Call history:** the same badge and line, for good: what was known when the call ended.
It's **information only**. Calls are never blocked, delayed or sent elsewhere, and a lookup that
fails or is slow only means nothing is shown (and one warning in the api log). Low-risk callers,
contacts, internal calls, withheld numbers and numbers outside the US show nothing.
It's off until you turn it on in **Settings → Spam check**.
## The two sources
You can use either one, or both. With both, IPQualityScore answers first, and the FTC list
answers when IPQualityScore isn't set up, fails, or has run out of lookups.
| | IPQualityScore | FTC complaint list |
|---|---|---|
| What it is | A live lookup service with a fraud score per number | Complaints people filed with the FTC about unwanted calls |
| Cost | 1,000 lookups a month free, then paid plans | Free |
| Sends your callers' numbers out | **Yes**, to IPQualityScore | No: the list is downloaded to this server |
| Likely spam | Flagged as a spammer, or fraud score 85+ | 3+ complaints in the window |
| Possible spam | Fraud score 75+, recent abuse, or risky | 1–2 complaints |
The FTC's reports aren't verified, and spammers fake real people's numbers: a number with
complaints may belong to someone whose number was faked. That's why the badge says what was
reported, not that the caller is a spammer.
## IPQualityScore
1. Sign up at <https://www.ipqualityscore.com/> and copy your API key from the account settings.
2. In **Settings → Spam check**, tick **Use IPQualityScore**, paste the key and press **Test**.
It shows how many lookups are left this billing period.
3. Tick **Spam check on** and **Save**. You're asked to confirm that callers' numbers will be
sent to IPQualityScore (see [Privacy](#privacy)).
Each number is looked up once and the answer kept: 14 days for possible and likely spam, 3 days
for the rest (**Keep IPQualityScore answers** changes these). Five phones ringing for the same
call cost one lookup.
**When the lookups run out**, SelfPhoneNG stops asking IPQualityScore and the dashboard says so
(**Dismiss** hides it until the next time). Checks use the FTC list meanwhile, if it's set up.
Every 6 hours it asks IPQualityScore whether lookups are back and carries on by itself. Saving a
new key carries on at once.
## FTC complaint list
1. Get a free key at <https://api.data.gov/signup/>.
2. In **Settings → Spam check**, tick **Use the FTC list**, paste the key, press **Test**, then
**Save** (with **Spam check on**).
The first download takes the last 30 days (**Keep complaints for**) and runs for about 7 hours:
api.data.gov allows a key 1,000 requests an hour, and there are 5,000 to 15,000 complaints a day.
The tab shows how far it got. After that it fetches what's new every 24 hours (**Download again
every**); **Download now** starts one at once. If the FTC refuses the key or is down, the tab
shows why and it tries again every minute from where it stopped.
## Privacy
- With **IPQualityScore**, the number of every outside caller who isn't a contact is sent to
IPQualityScore, a company in the US. If your privacy notice lists who you share data with, add
it. In the EU and UK the GDPR applies: you need a lawful basis (usually legitimate interest in
protecting staff from fraud) and a data processing agreement with IPQualityScore.
- With the **FTC list** only, nothing about your calls leaves the server.
- Only a number that's ringing one of the user's own phones is looked up: nobody can use the
browser phone to look up numbers that never called.
- The keys are stored encrypted and never shown again; the audit log hides them.
## If nothing shows
- Is **Spam check on**, and is the caller in someone's contacts (contacts always win)?
- Check the api log for `spam check failed` (it says which source and why) or
`FTC complaint download failed`:
```sh
docker compose logs api | grep -iE 'spam|ftc'
```
- Desk phones only show the label when the answer is quick. A first call from a number may show
nothing there while the browser phone shows it; the next call uses the kept answer.
+1 -1
View File
@@ -9,7 +9,7 @@ Legend: 📖 read · 🔨 build · ✅ done when
---
## Backlog (owner's order, 2026-10-21)
0. **Spam check** (owner, 2026-10-27: SD1–SD4 done; SD5 next), tasks SD1–SD5
0. **Spam check** (owner, 2026-10-27: SD1–SD5 done; the owner's real-key checks are left), tasks SD1–SD5
in [14 §9](14-spam.md#9-order-of-work): IPQualityScore and the FTC complaint list for outside
callers who aren't in anyone's contacts; information only, off by default.
0. **Mobile app** (owner, 2026-10-26: next), tasks MB1–MB8 in [13 §5](13-mobile.md#5-order-of-work):
+15 -1
View File
@@ -236,7 +236,7 @@ log masks it. Validated with zod (`packages/shared`).
| SD2 | §3: pause, events, the `spam_quota` warning and Dismiss, `spamQuotaCheck` | an exhausted fake pauses, falls back to the FTC list, warns once and resumes; the owner's real key shows the credit call is free. Done but the real-key check, see §9.2 |
| SD3 | §2.2 sync: paging, pacing, the saved cursor, the window, Sync now | a fake FTC API with 3 days of data syncs, resumes after a 429 and drops old complaints; one real sync with the owner's key. Done but the real sync, see §9.3 |
| SD4 | §5: `/api/me/caller-check`, the softphone badge and line, the desk phone labels | an unknown number rings the softphone and shows its answer; a contact or a low risk shows nothing; a desk phone shows "SPAM LIKELY" when the answer is in time. Done but a real call, see §9.4 |
| SD5 | §6 call history; the admin guide page | a checked call keeps its badge in call history after its cache expires |
| SD5 | §6 call history; the admin guide page | a checked call keeps its badge in call history after its cache expires. Done, see §9.5 |
### 9.1 As built: SD1
@@ -339,6 +339,20 @@ log masks it. Validated with zod (`packages/shared`).
a late answer then the cache, custom labels from the FTC list), `SpamNote.test.ts` (the line).
- Not checked: a real call in the browser and on a desk phone.
### 9.5 As built: SD5
- `aggregateCall` (04 §5.4) runs `checkNumber(…, { cachedOnly: true })` for inbound calls and
writes `spam_risk` and `spam_info` (`{ provider, info }`) with the row. A low answer is kept
too (the FTC list answers low for a number nobody reported); null means not checked.
- `CallDto.spam` on `/api/calls`, `/api/calls/:id` and `/api/me/calls`: null when the viewer's
books (theirs and the company's) name the number, or when the books can't be read.
- Calls pages: `SpamNote` under the caller, on `/me/calls` only for calls to me that the browser
doesn't know as a contact. The CSV export has no spam columns.
- Admin guide: `docs/admin-guide/spam-check.md`.
- Tests: `test/spam.test.ts` (four calls aggregated: IPQualityScore's answer and the FTC's kept
after the cache is emptied, a contact's call shows nothing, an unreported number low, and no
request out).
Later, not now:
- **Report as spam** (owner, 2026-10-27: not now): a button in call history that reports the
number to a provider that takes reports (OpenSpam has `POST /api/report`).
+3
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// Call history (02 calls, call_extensions; 04 §5.4, §7.3).
import { z } from 'zod';
import type { SpamCheck } from './spam.ts';
export const CALL_DIRECTIONS = ['inbound', 'outbound', 'internal'] as const;
export const CALL_STATUSES = [
@@ -43,6 +44,8 @@ export type CallDto = {
answeredBy: { id: string; number: string; name: string } | null;
talkSeconds: number;
recordingId: string | null;
/** 14 §6: what the spam check knew as it ended; null when not checked or in the viewer's books. */
spam: SpamCheck | null;
};
/** One raw CDR leg, for the expanded row. */