65 lines
3.3 KiB
TypeScript
65 lines
3.3 KiB
TypeScript
/**
|
|
* Subject moderation for the "Create" feature. This is a children's app and the subject text feeds an
|
|
* image generator, so we are deliberately conservative: sanitize → allowlist (auto-approve) →
|
|
* blocklist (reject) → otherwise hold for admin review.
|
|
*/
|
|
|
|
export type ModerationDecision =
|
|
| { decision: "allowlist"; subject: string }
|
|
| { decision: "review"; subject: string }
|
|
| { decision: "blocked"; subject: string; reason: string };
|
|
|
|
/** Clean raw input to a single short, plain phrase. Returns null if nothing usable remains. */
|
|
export function sanitizeSubject(raw: string): string | null {
|
|
if (typeof raw !== "string") return null;
|
|
const cleaned = raw
|
|
.replace(/[\r\n\t]+/g, " ")
|
|
.replace(/[^a-zA-Z\- ]+/g, "") // letters, spaces, hyphens only — strips digits/punctuation/prompt tokens
|
|
.replace(/\s+/g, " ")
|
|
.trim()
|
|
.slice(0, 30);
|
|
if (cleaned.length < 2) return null;
|
|
// Title-case for display ("zebra" -> "Zebra").
|
|
return cleaned.replace(/\b\w/g, (c) => c.toUpperCase());
|
|
}
|
|
|
|
// Curated safe subjects (lowercase). Auto-approved. Extend freely.
|
|
const ALLOWLIST = new Set<string>([
|
|
// animals
|
|
"cat", "dog", "puppy", "kitten", "rabbit", "bunny", "fox", "bear", "panda", "lion", "tiger",
|
|
"elephant", "giraffe", "zebra", "monkey", "deer", "horse", "pony", "cow", "pig", "sheep", "goat",
|
|
"duck", "chicken", "owl", "bird", "penguin", "fish", "shark", "whale", "dolphin", "octopus", "crab",
|
|
"turtle", "frog", "snail", "bee", "butterfly", "ladybug", "snake", "dinosaur", "dragon", "unicorn",
|
|
"hedgehog", "squirrel", "mouse", "koala", "kangaroo", "crocodile", "leopard", "snail", "starfish",
|
|
// nature
|
|
"tree", "flower", "rose", "sunflower", "leaf", "mushroom", "cactus", "cloud", "sun", "moon", "star",
|
|
"rainbow", "mountain", "apple", "banana", "strawberry", "cherry", "grapes", "carrot", "pumpkin",
|
|
// objects / vehicles / fun
|
|
"house", "castle", "boat", "sailboat", "car", "truck", "train", "bus", "plane", "rocket", "robot",
|
|
"balloon", "kite", "umbrella", "cup", "mug", "hat", "shirt", "dress", "sock", "shoe", "bowtie",
|
|
"bow tie", "gift", "present", "ball", "football", "basketball", "trophy", "crown", "heart", "star",
|
|
"square", "triangle", "circle", "cake", "cupcake", "cookie", "ice cream", "lollipop", "snowman",
|
|
"teddy bear", "ghost", "pumpkin", "guitar", "drum", "book", "pencil", "key", "clock", "lamp",
|
|
]);
|
|
|
|
// Obvious unsafe terms (substring match, after sanitize). Backstop, not exhaustive.
|
|
const BLOCKLIST: string[] = [
|
|
"gun", "knife", "weapon", "sword", "blood", "gore", "kill", "dead", "death", "corpse", "drug",
|
|
"beer", "wine", "alcohol", "cigarette", "smoke", "nude", "naked", "sex", "sexy", "porn", "kiss",
|
|
"hate", "nazi", "racist", "suicide", "hang", "noose", "bomb", "grenade", "war", "shoot", "knive",
|
|
];
|
|
|
|
export function moderateSubject(raw: string): ModerationDecision {
|
|
const subject = sanitizeSubject(raw);
|
|
if (!subject) return { decision: "blocked", subject: "", reason: "empty" };
|
|
const lower = subject.toLowerCase();
|
|
|
|
const bad = BLOCKLIST.find((w) => lower.includes(w));
|
|
if (bad) return { decision: "blocked", subject, reason: `term "${bad}"` };
|
|
|
|
if (ALLOWLIST.has(lower)) return { decision: "allowlist", subject };
|
|
|
|
// Not obviously safe and not obviously bad → let an admin decide.
|
|
return { decision: "review", subject };
|
|
}
|