From f6ef399d92b8479572a8a3d90a78651e43b6b0aa Mon Sep 17 00:00:00 2001 From: iamdoubz <> Date: Thu, 5 Nov 2026 16:08:31 -0600 Subject: [PATCH] Add uploads, background extraction worker, matching and publishing --- parentcalendar/scans.py | 235 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 parentcalendar/scans.py diff --git a/parentcalendar/scans.py b/parentcalendar/scans.py new file mode 100644 index 0000000..3eba721 --- /dev/null +++ b/parentcalendar/scans.py @@ -0,0 +1,235 @@ +"""Uploads, the background extraction job, matching against existing events, and publishing.""" +import hashlib +import io +import json +import logging +import threading +import time +import uuid +from datetime import date, datetime, timedelta, timezone +from difflib import SequenceMatcher + +from PIL import Image, ImageOps + +from parentcalendar import config, db, extraction + +log = logging.getLogger("parentcalendar") +MATCH_FIELDS = ("date", "end_date", "all_day", "start_time", "end_time", "title", "notes", "location", "kind") +SURE_MATCH = 0.85 # title similarity; below this a match is shown but flagged + + +class UploadError(ValueError): + pass + + +def new_uid(): + return f"{uuid.uuid4().hex}@{config.UID_DOMAIN}" + + +def norm(s): + return "".join(c for c in (s or "").lower() if c.isalnum()) + + +def pages_from_upload(data): + """Images or PDFs in, PIL pages out. Anything Pillow or PDFium can't open is rejected.""" + if len(data) > config.MAX_UPLOAD_BYTES: + raise UploadError(f"file is larger than {config.MAX_UPLOAD_BYTES // 2**20} MB") + if data[:5] == b"%PDF-": + import pypdfium2 as pdfium + try: + pdf = pdfium.PdfDocument(data) + except pdfium.PdfiumError as e: + raise UploadError(f"unreadable PDF: {e}") from e + if len(pdf) > 10: + raise UploadError("PDFs are limited to 10 pages") + return [pdf[i].render(scale=2).to_pil().convert("RGB") for i in range(len(pdf))] + try: + img = Image.open(io.BytesIO(data)) + img.load() + except (OSError, Image.DecompressionBombError) as e: + raise UploadError("not an image or PDF") from e + return [ImageOps.exif_transpose(img).convert("RGB")] + + +def save_upload(conn, source_id, hint_month, files): + """files: list of bytes. Re-encodes every page as JPEG on disk and queues a job. + The same files for the same source and month return the existing scan instead (no second model run).""" + digest = hashlib.sha256(b"".join(hashlib.sha256(f).digest() for f in files)).hexdigest() + existing = db.one(conn, "SELECT * FROM scan WHERE image_hash = ? AND source_id = ? AND hint_month = ? " + "AND status NOT IN ('failed', 'abandoned') ORDER BY id DESC", (digest, source_id, hint_month)) + if existing: + return existing["id"] + pages = [p for f in files for p in pages_from_upload(f)] + if not pages: + raise UploadError("no pages") + with conn: + scan_id = conn.execute("INSERT INTO scan (source_id, hint_month, image_hash) VALUES (?, ?, ?)", + (source_id, hint_month, digest)).lastrowid + keys = [] + for n, page in enumerate(pages, 1): + key = f"{scan_id}-{n}.jpg" + page.save(config.IMAGES_DIR / key, "JPEG", quality=90) + keys.append(key) + conn.execute("UPDATE scan SET image_keys = ? WHERE id = ?", (json.dumps(keys), scan_id)) + conn.execute("INSERT INTO job (scan_id) VALUES (?)", (scan_id,)) + return scan_id + + +def processed_key(key): + return key.replace(".jpg", "-p.jpg") + + +def process_scan(conn, scan_id): + """Run extraction on every page and store the results as drafts.""" + scan = db.one(conn, "SELECT * FROM scan WHERE id = ?", (scan_id,)) + source = db.one(conn, "SELECT * FROM source WHERE id = ?", (scan["source_id"],)) + child_ids = [r[0] for r in conn.execute("SELECT child_id FROM source_child WHERE source_id = ?", (source["id"],))] + year, month = map(int, scan["hint_month"].split("-")) + hint = {"year": year, "month": month, "layout": source["layout_type"], "filter": source["applies_filter"]} + ignore = extraction.IGNORE | {w.strip().lower() for w in source["ignore_labels"] if w.strip()} + runs = [] + for key in scan["image_keys"]: + img = extraction.load_image(config.IMAGES_DIR / key) + run = extraction.extract_image(img, hint, double_read=source["double_read"], ignore=ignore) + run.pop("image").save(config.IMAGES_DIR / processed_key(key), "JPEG", quality=90) + runs.append(run) + with conn: + conn.execute("DELETE FROM event WHERE scan_id = ? AND status = 'draft'", (scan_id,)) # a retry starts clean + conn.execute("DELETE FROM menu_item WHERE scan_id = ?", (scan_id,)) + dates = [] + for page, run in enumerate(runs, 1): + r = run["result"] + dates += [r["date_range"].get("start"), r["date_range"].get("end")] + for e in r["events"]: + event_id = conn.execute( + "INSERT INTO event (uid, scan_id, source_id, date, end_date, all_day, start_time, end_time, title," + " notes, location, kind, applies, include, page, cell_bbox, source_text, confidence, flags)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (new_uid(), scan_id, source["id"], e["date"], e.get("end_date") if e.get("end_date") != e["date"] else None, + e["all_day"], e.get("start_time"), e.get("end_time"), e["title"], e.get("notes"), e.get("location"), + e["kind"], e.get("applies", True), e.get("applies", True), page, json.dumps(e.get("cell_bbox")), + e.get("source_text"), e.get("confidence"), json.dumps(e.get("flags", [])))).lastrowid + conn.executemany("INSERT INTO event_child VALUES (?, ?)", [(event_id, c) for c in child_ids]) + for m in r["menu_items"]: + conn.execute( + "INSERT OR REPLACE INTO menu_item (scan_id, source_id, date, items, no_lunch, page, cell_bbox," + " confidence, flags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (scan_id, source["id"], m["date"], json.dumps(m["items"]), m["no_lunch"], page, + json.dumps(m.get("cell_bbox")), m.get("confidence"), json.dumps(m.get("flags", [])))) + dates = sorted(d for d in dates if d) + tokens = sum(r["usage"]["input_tokens"] + r["usage"]["output_tokens"] for r in runs) + conn.execute("UPDATE scan SET status = 'parsed', error = NULL, date_start = ?, date_end = ?, model = ?," + " cost_tokens = ?, seconds = ? WHERE id = ?", + (dates[0] if dates else None, dates[-1] if dates else None, runs[0]["model"] if runs else None, + tokens, round(sum(r["seconds"] for r in runs), 1), scan_id)) + match_drafts(conn, scan_id) + + +def overlaps(a, b): + return a["date"] <= (b["end_date"] or b["date"]) and b["date"] <= (a["end_date"] or a["date"]) + + +def match_drafts(conn, scan_id): + """Link each draft to the source's confirmed event it most likely is, so publishing updates that event + (same UID) instead of adding a duplicate. Same source, overlapping dates, similar title.""" + drafts = db.all_dicts(conn.execute("SELECT * FROM event WHERE scan_id = ? AND status = 'draft'", (scan_id,))) + if not drafts: + return + existing = db.all_dicts(conn.execute( + "SELECT * FROM event WHERE source_id = ? AND status = 'confirmed' AND date <= ? AND COALESCE(end_date, date) >= ?", + (drafts[0]["source_id"], max(d["end_date"] or d["date"] for d in drafts), min(d["date"] for d in drafts)))) + pairs = sorted(((SequenceMatcher(None, norm(d["title"]), norm(e["title"])).ratio(), d["id"], e["id"]) + for d in drafts for e in existing if overlaps(d, e)), reverse=True) + by_id = {d["id"]: d for d in drafts} | {e["id"]: e for e in existing} + used = set() + for score, did, eid in pairs: + if score < 0.6 or did in used or eid in used: + continue + used |= {did, eid} + d, e = by_id[did], by_id[eid] + same = all(d[f] == e[f] for f in MATCH_FIELDS) + flags = d["flags"] + ([] if score >= SURE_MATCH else [f"might be the same as \"{e['title']}\" on {e['date']}"]) + conn.execute("UPDATE event SET replaces_id = ?, match = ?, flags = ? WHERE id = ?", + (eid, "unchanged" if same else "updated", json.dumps(flags), did)) + conn.execute("UPDATE event SET match = 'new' WHERE scan_id = ? AND status = 'draft' AND match IS NULL", (scan_id,)) + + +def publish(conn, scan_id): + """Checked drafts become confirmed events; a draft matched to an existing event updates it in place.""" + # Until a scan is published, every event carrying its scan_id is one of its drafts + # (status 'deleted' here means deleted during review). + drafts = db.all_dicts(conn.execute("SELECT * FROM event WHERE scan_id = ?", (scan_id,))) + with conn: + for d in drafts: + if d["status"] == "deleted" or not d["include"]: + conn.execute("DELETE FROM event WHERE id = ?", (d["id"],)) + elif d["replaces_id"]: + old = db.one(conn, "SELECT * FROM event WHERE id = ?", (d["replaces_id"],)) + if any(d[f] != old[f] for f in MATCH_FIELDS) or old["status"] != "confirmed": + conn.execute( + "UPDATE event SET " + ", ".join(f"{f} = ?" for f in MATCH_FIELDS) + + ", status = 'confirmed', sequence = sequence + 1, updated_at = strftime('%Y-%m-%d %H:%M:%f', 'now')," + " scan_id = ?, page = ?, cell_bbox = ?, source_text = ? WHERE id = ?", + [d[f] for f in MATCH_FIELDS] + [scan_id, d["page"], json.dumps(d["cell_bbox"]), + d["source_text"], old["id"]]) + conn.execute("DELETE FROM event_child WHERE event_id = ?", (old["id"],)) + conn.execute("INSERT INTO event_child SELECT ?, child_id FROM event_child WHERE event_id = ?", + (old["id"], d["id"])) + conn.execute("DELETE FROM event WHERE id = ?", (d["id"],)) + else: + conn.execute("UPDATE event SET status = 'confirmed', updated_at = strftime('%Y-%m-%d %H:%M:%f', 'now') WHERE id = ?", (d["id"],)) + conn.execute("UPDATE scan SET status = 'published', published_at = CURRENT_TIMESTAMP WHERE id = ?", (scan_id,)) + + +def clean_old_images(conn): + """Delete page images IMAGE_RETENTION_DAYS after a scan was published or abandoned.""" + cutoff = (datetime.now(timezone.utc) - timedelta(days=config.IMAGE_RETENTION_DAYS)).strftime("%Y-%m-%d %H:%M:%S") + old = db.all_dicts(conn.execute( + "SELECT id, image_keys FROM scan WHERE image_keys != '[]' AND ((status = 'published' AND published_at < ?)" + " OR (status = 'abandoned' AND created_at < ?))", (cutoff, cutoff))) + for scan in old: + for key in scan["image_keys"]: + for name in (key, processed_key(key)): + (config.IMAGES_DIR / name).unlink(missing_ok=True) + with conn: + conn.execute("UPDATE scan SET image_keys = '[]' WHERE id = ?", (scan["id"],)) + + +def run_worker(stop): + """Background thread: one job at a time, so the GPU only ever has one page in flight.""" + with db.connect() as conn: + conn.execute("UPDATE job SET status = 'queued' WHERE status = 'running'") # interrupted by a restart + conn.commit() + last_cleanup = 0.0 + while not stop.is_set(): + conn = db.connect() + try: + if time.time() - last_cleanup > 3600: + clean_old_images(conn) + last_cleanup = time.time() + job = conn.execute("UPDATE job SET status = 'running', attempts = attempts + 1 WHERE id =" + " (SELECT id FROM job WHERE status = 'queued' ORDER BY id LIMIT 1) RETURNING id, scan_id" + ).fetchone() + conn.commit() + if job is None: + stop.wait(2) + continue + try: + process_scan(conn, job["scan_id"]) + status, error = "done", None + except Exception as e: # any failure marks this scan failed; the worker keeps serving the queue + log.exception("scan %s failed", job["scan_id"]) + status, error = "failed", f"{type(e).__name__}: {e}" + with conn: + conn.execute("UPDATE job SET status = ?, error = ? WHERE id = ?", (status, error, job["id"])) + if error: + conn.execute("UPDATE scan SET status = 'failed', error = ? WHERE id = ?", (error, job["scan_id"])) + finally: + conn.close() + + +def start_worker(): + stop = threading.Event() + thread = threading.Thread(target=run_worker, args=(stop,), name="extraction-worker", daemon=True) + thread.start() + return stop, thread