Add uploads, background extraction worker, matching and publishing

This commit is contained in:
iamdoubz committed 2026-11-05 16:08:31 -06:00
1 parent fe0ddaf533
commit f6ef399d92
1 file changed
+235
+235
View File
@@ -0,0 +1,235 @@
"""Uploads, the background extraction job, matching against existing events, and publishing."""
import hashlib
import io
import json
import logging
import threading
import time
import uuid
from datetime import date, datetime, timedelta, timezone
from difflib import SequenceMatcher
from PIL import Image, ImageOps
from parentcalendar import config, db, extraction
log = logging.getLogger("parentcalendar")
MATCH_FIELDS = ("date", "end_date", "all_day", "start_time", "end_time", "title", "notes", "location", "kind")
SURE_MATCH = 0.85 # title similarity; below this a match is shown but flagged
class UploadError(ValueError):
pass
def new_uid():
return f"{uuid.uuid4().hex}@{config.UID_DOMAIN}"
def norm(s):
return "".join(c for c in (s or "").lower() if c.isalnum())
def pages_from_upload(data):
"""Images or PDFs in, PIL pages out. Anything Pillow or PDFium can't open is rejected."""
if len(data) > config.MAX_UPLOAD_BYTES:
raise UploadError(f"file is larger than {config.MAX_UPLOAD_BYTES // 2**20} MB")
if data[:5] == b"%PDF-":
import pypdfium2 as pdfium
try:
pdf = pdfium.PdfDocument(data)
except pdfium.PdfiumError as e:
raise UploadError(f"unreadable PDF: {e}") from e
if len(pdf) > 10:
raise UploadError("PDFs are limited to 10 pages")
return [pdf[i].render(scale=2).to_pil().convert("RGB") for i in range(len(pdf))]
try:
img = Image.open(io.BytesIO(data))
img.load()
except (OSError, Image.DecompressionBombError) as e:
raise UploadError("not an image or PDF") from e
return [ImageOps.exif_transpose(img).convert("RGB")]
def save_upload(conn, source_id, hint_month, files):
"""files: list of bytes. Re-encodes every page as JPEG on disk and queues a job.
The same files for the same source and month return the existing scan instead (no second model run)."""
digest = hashlib.sha256(b"".join(hashlib.sha256(f).digest() for f in files)).hexdigest()
existing = db.one(conn, "SELECT * FROM scan WHERE image_hash = ? AND source_id = ? AND hint_month = ? "
"AND status NOT IN ('failed', 'abandoned') ORDER BY id DESC", (digest, source_id, hint_month))
if existing:
return existing["id"]
pages = [p for f in files for p in pages_from_upload(f)]
if not pages:
raise UploadError("no pages")
with conn:
scan_id = conn.execute("INSERT INTO scan (source_id, hint_month, image_hash) VALUES (?, ?, ?)",
(source_id, hint_month, digest)).lastrowid
keys = []
for n, page in enumerate(pages, 1):
key = f"{scan_id}-{n}.jpg"
page.save(config.IMAGES_DIR / key, "JPEG", quality=90)
keys.append(key)
conn.execute("UPDATE scan SET image_keys = ? WHERE id = ?", (json.dumps(keys), scan_id))
conn.execute("INSERT INTO job (scan_id) VALUES (?)", (scan_id,))
return scan_id
def processed_key(key):
return key.replace(".jpg", "-p.jpg")
def process_scan(conn, scan_id):
"""Run extraction on every page and store the results as drafts."""
scan = db.one(conn, "SELECT * FROM scan WHERE id = ?", (scan_id,))
source = db.one(conn, "SELECT * FROM source WHERE id = ?", (scan["source_id"],))
child_ids = [r[0] for r in conn.execute("SELECT child_id FROM source_child WHERE source_id = ?", (source["id"],))]
year, month = map(int, scan["hint_month"].split("-"))
hint = {"year": year, "month": month, "layout": source["layout_type"], "filter": source["applies_filter"]}
ignore = extraction.IGNORE | {w.strip().lower() for w in source["ignore_labels"] if w.strip()}
runs = []
for key in scan["image_keys"]:
img = extraction.load_image(config.IMAGES_DIR / key)
run = extraction.extract_image(img, hint, double_read=source["double_read"], ignore=ignore)
run.pop("image").save(config.IMAGES_DIR / processed_key(key), "JPEG", quality=90)
runs.append(run)
with conn:
conn.execute("DELETE FROM event WHERE scan_id = ? AND status = 'draft'", (scan_id,)) # a retry starts clean
conn.execute("DELETE FROM menu_item WHERE scan_id = ?", (scan_id,))
dates = []
for page, run in enumerate(runs, 1):
r = run["result"]
dates += [r["date_range"].get("start"), r["date_range"].get("end")]
for e in r["events"]:
event_id = conn.execute(
"INSERT INTO event (uid, scan_id, source_id, date, end_date, all_day, start_time, end_time, title,"
" notes, location, kind, applies, include, page, cell_bbox, source_text, confidence, flags)"
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(new_uid(), scan_id, source["id"], e["date"], e.get("end_date") if e.get("end_date") != e["date"] else None,
e["all_day"], e.get("start_time"), e.get("end_time"), e["title"], e.get("notes"), e.get("location"),
e["kind"], e.get("applies", True), e.get("applies", True), page, json.dumps(e.get("cell_bbox")),
e.get("source_text"), e.get("confidence"), json.dumps(e.get("flags", [])))).lastrowid
conn.executemany("INSERT INTO event_child VALUES (?, ?)", [(event_id, c) for c in child_ids])
for m in r["menu_items"]:
conn.execute(
"INSERT OR REPLACE INTO menu_item (scan_id, source_id, date, items, no_lunch, page, cell_bbox,"
" confidence, flags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(scan_id, source["id"], m["date"], json.dumps(m["items"]), m["no_lunch"], page,
json.dumps(m.get("cell_bbox")), m.get("confidence"), json.dumps(m.get("flags", []))))
dates = sorted(d for d in dates if d)
tokens = sum(r["usage"]["input_tokens"] + r["usage"]["output_tokens"] for r in runs)
conn.execute("UPDATE scan SET status = 'parsed', error = NULL, date_start = ?, date_end = ?, model = ?,"
" cost_tokens = ?, seconds = ? WHERE id = ?",
(dates[0] if dates else None, dates[-1] if dates else None, runs[0]["model"] if runs else None,
tokens, round(sum(r["seconds"] for r in runs), 1), scan_id))
match_drafts(conn, scan_id)
def overlaps(a, b):
return a["date"] <= (b["end_date"] or b["date"]) and b["date"] <= (a["end_date"] or a["date"])
def match_drafts(conn, scan_id):
"""Link each draft to the source's confirmed event it most likely is, so publishing updates that event
(same UID) instead of adding a duplicate. Same source, overlapping dates, similar title."""
drafts = db.all_dicts(conn.execute("SELECT * FROM event WHERE scan_id = ? AND status = 'draft'", (scan_id,)))
if not drafts:
return
existing = db.all_dicts(conn.execute(
"SELECT * FROM event WHERE source_id = ? AND status = 'confirmed' AND date <= ? AND COALESCE(end_date, date) >= ?",
(drafts[0]["source_id"], max(d["end_date"] or d["date"] for d in drafts), min(d["date"] for d in drafts))))
pairs = sorted(((SequenceMatcher(None, norm(d["title"]), norm(e["title"])).ratio(), d["id"], e["id"])
for d in drafts for e in existing if overlaps(d, e)), reverse=True)
by_id = {d["id"]: d for d in drafts} | {e["id"]: e for e in existing}
used = set()
for score, did, eid in pairs:
if score < 0.6 or did in used or eid in used:
continue
used |= {did, eid}
d, e = by_id[did], by_id[eid]
same = all(d[f] == e[f] for f in MATCH_FIELDS)
flags = d["flags"] + ([] if score >= SURE_MATCH else [f"might be the same as \"{e['title']}\" on {e['date']}"])
conn.execute("UPDATE event SET replaces_id = ?, match = ?, flags = ? WHERE id = ?",
(eid, "unchanged" if same else "updated", json.dumps(flags), did))
conn.execute("UPDATE event SET match = 'new' WHERE scan_id = ? AND status = 'draft' AND match IS NULL", (scan_id,))
def publish(conn, scan_id):
"""Checked drafts become confirmed events; a draft matched to an existing event updates it in place."""
# Until a scan is published, every event carrying its scan_id is one of its drafts
# (status 'deleted' here means deleted during review).
drafts = db.all_dicts(conn.execute("SELECT * FROM event WHERE scan_id = ?", (scan_id,)))
with conn:
for d in drafts:
if d["status"] == "deleted" or not d["include"]:
conn.execute("DELETE FROM event WHERE id = ?", (d["id"],))
elif d["replaces_id"]:
old = db.one(conn, "SELECT * FROM event WHERE id = ?", (d["replaces_id"],))
if any(d[f] != old[f] for f in MATCH_FIELDS) or old["status"] != "confirmed":
conn.execute(
"UPDATE event SET " + ", ".join(f"{f} = ?" for f in MATCH_FIELDS) +
", status = 'confirmed', sequence = sequence + 1, updated_at = strftime('%Y-%m-%d %H:%M:%f', 'now'),"
" scan_id = ?, page = ?, cell_bbox = ?, source_text = ? WHERE id = ?",
[d[f] for f in MATCH_FIELDS] + [scan_id, d["page"], json.dumps(d["cell_bbox"]),
d["source_text"], old["id"]])
conn.execute("DELETE FROM event_child WHERE event_id = ?", (old["id"],))
conn.execute("INSERT INTO event_child SELECT ?, child_id FROM event_child WHERE event_id = ?",
(old["id"], d["id"]))
conn.execute("DELETE FROM event WHERE id = ?", (d["id"],))
else:
conn.execute("UPDATE event SET status = 'confirmed', updated_at = strftime('%Y-%m-%d %H:%M:%f', 'now') WHERE id = ?", (d["id"],))
conn.execute("UPDATE scan SET status = 'published', published_at = CURRENT_TIMESTAMP WHERE id = ?", (scan_id,))
def clean_old_images(conn):
"""Delete page images IMAGE_RETENTION_DAYS after a scan was published or abandoned."""
cutoff = (datetime.now(timezone.utc) - timedelta(days=config.IMAGE_RETENTION_DAYS)).strftime("%Y-%m-%d %H:%M:%S")
old = db.all_dicts(conn.execute(
"SELECT id, image_keys FROM scan WHERE image_keys != '[]' AND ((status = 'published' AND published_at < ?)"
" OR (status = 'abandoned' AND created_at < ?))", (cutoff, cutoff)))
for scan in old:
for key in scan["image_keys"]:
for name in (key, processed_key(key)):
(config.IMAGES_DIR / name).unlink(missing_ok=True)
with conn:
conn.execute("UPDATE scan SET image_keys = '[]' WHERE id = ?", (scan["id"],))
def run_worker(stop):
"""Background thread: one job at a time, so the GPU only ever has one page in flight."""
with db.connect() as conn:
conn.execute("UPDATE job SET status = 'queued' WHERE status = 'running'") # interrupted by a restart
conn.commit()
last_cleanup = 0.0
while not stop.is_set():
conn = db.connect()
try:
if time.time() - last_cleanup > 3600:
clean_old_images(conn)
last_cleanup = time.time()
job = conn.execute("UPDATE job SET status = 'running', attempts = attempts + 1 WHERE id ="
" (SELECT id FROM job WHERE status = 'queued' ORDER BY id LIMIT 1) RETURNING id, scan_id"
).fetchone()
conn.commit()
if job is None:
stop.wait(2)
continue
try:
process_scan(conn, job["scan_id"])
status, error = "done", None
except Exception as e: # any failure marks this scan failed; the worker keeps serving the queue
log.exception("scan %s failed", job["scan_id"])
status, error = "failed", f"{type(e).__name__}: {e}"
with conn:
conn.execute("UPDATE job SET status = ?, error = ? WHERE id = ?", (status, error, job["id"]))
if error:
conn.execute("UPDATE scan SET status = 'failed', error = ? WHERE id = ?", (error, job["scan_id"]))
finally:
conn.close()
def start_worker():
stop = threading.Event()
thread = threading.Thread(target=run_worker, args=(stop,), name="extraction-worker", daemon=True)
thread.start()
return stop, thread