236 lines
12 KiB
Python
236 lines
12 KiB
Python
"""Uploads, the background extraction job, matching against existing events, and publishing."""
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import logging
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from difflib import SequenceMatcher
|
|
|
|
from PIL import Image, ImageOps
|
|
|
|
from parentcalendar import config, db, extraction
|
|
|
|
log = logging.getLogger("parentcalendar")
|
|
MATCH_FIELDS = ("date", "end_date", "all_day", "start_time", "end_time", "title", "notes", "location", "kind")
|
|
SURE_MATCH = 0.85 # title similarity; below this a match is shown but flagged
|
|
|
|
|
|
class UploadError(ValueError):
|
|
pass
|
|
|
|
|
|
def new_uid():
|
|
return f"{uuid.uuid4().hex}@{config.UID_DOMAIN}"
|
|
|
|
|
|
def norm(s):
|
|
return "".join(c for c in (s or "").lower() if c.isalnum())
|
|
|
|
|
|
def pages_from_upload(data):
|
|
"""Images or PDFs in, PIL pages out. Anything Pillow or PDFium can't open is rejected."""
|
|
if len(data) > config.MAX_UPLOAD_BYTES:
|
|
raise UploadError(f"file is larger than {config.MAX_UPLOAD_BYTES // 2**20} MB")
|
|
if data[:5] == b"%PDF-":
|
|
import pypdfium2 as pdfium
|
|
try:
|
|
pdf = pdfium.PdfDocument(data)
|
|
except pdfium.PdfiumError as e:
|
|
raise UploadError(f"unreadable PDF: {e}") from e
|
|
if len(pdf) > 10:
|
|
raise UploadError("PDFs are limited to 10 pages")
|
|
return [pdf[i].render(scale=2).to_pil().convert("RGB") for i in range(len(pdf))]
|
|
try:
|
|
img = Image.open(io.BytesIO(data))
|
|
img.load()
|
|
except (OSError, Image.DecompressionBombError) as e:
|
|
raise UploadError("not an image or PDF") from e
|
|
return [ImageOps.exif_transpose(img).convert("RGB")]
|
|
|
|
|
|
def save_upload(conn, source_id, hint_month, files):
|
|
"""files: list of bytes. Re-encodes every page as JPEG on disk and queues a job.
|
|
The same files for the same source and month return the existing scan instead (no second model run)."""
|
|
digest = hashlib.sha256(b"".join(hashlib.sha256(f).digest() for f in files)).hexdigest()
|
|
existing = db.one(conn, "SELECT * FROM scan WHERE image_hash = ? AND source_id = ? AND hint_month = ? "
|
|
"AND status NOT IN ('failed', 'abandoned') ORDER BY id DESC", (digest, source_id, hint_month))
|
|
if existing:
|
|
return existing["id"]
|
|
pages = [p for f in files for p in pages_from_upload(f)]
|
|
if not pages:
|
|
raise UploadError("no pages")
|
|
with conn:
|
|
scan_id = conn.execute("INSERT INTO scan (source_id, hint_month, image_hash) VALUES (?, ?, ?)",
|
|
(source_id, hint_month, digest)).lastrowid
|
|
keys = []
|
|
for n, page in enumerate(pages, 1):
|
|
key = f"{scan_id}-{n}.jpg"
|
|
page.save(config.IMAGES_DIR / key, "JPEG", quality=90)
|
|
keys.append(key)
|
|
conn.execute("UPDATE scan SET image_keys = ? WHERE id = ?", (json.dumps(keys), scan_id))
|
|
conn.execute("INSERT INTO job (scan_id) VALUES (?)", (scan_id,))
|
|
return scan_id
|
|
|
|
|
|
def processed_key(key):
|
|
return key.replace(".jpg", "-p.jpg")
|
|
|
|
|
|
def process_scan(conn, scan_id):
|
|
"""Run extraction on every page and store the results as drafts."""
|
|
scan = db.one(conn, "SELECT * FROM scan WHERE id = ?", (scan_id,))
|
|
source = db.one(conn, "SELECT * FROM source WHERE id = ?", (scan["source_id"],))
|
|
child_ids = [r[0] for r in conn.execute("SELECT child_id FROM source_child WHERE source_id = ?", (source["id"],))]
|
|
year, month = map(int, scan["hint_month"].split("-"))
|
|
hint = {"year": year, "month": month, "layout": source["layout_type"], "filter": source["applies_filter"]}
|
|
ignore = extraction.IGNORE | {w.strip().lower() for w in source["ignore_labels"] if w.strip()}
|
|
runs = []
|
|
for key in scan["image_keys"]:
|
|
img = extraction.load_image(config.IMAGES_DIR / key)
|
|
run = extraction.extract_image(img, hint, double_read=source["double_read"], ignore=ignore)
|
|
run.pop("image").save(config.IMAGES_DIR / processed_key(key), "JPEG", quality=90)
|
|
runs.append(run)
|
|
with conn:
|
|
conn.execute("DELETE FROM event WHERE scan_id = ? AND status = 'draft'", (scan_id,)) # a retry starts clean
|
|
conn.execute("DELETE FROM menu_item WHERE scan_id = ?", (scan_id,))
|
|
dates = []
|
|
for page, run in enumerate(runs, 1):
|
|
r = run["result"]
|
|
dates += [r["date_range"].get("start"), r["date_range"].get("end")]
|
|
for e in r["events"]:
|
|
event_id = conn.execute(
|
|
"INSERT INTO event (uid, scan_id, source_id, date, end_date, all_day, start_time, end_time, title,"
|
|
" notes, location, kind, applies, include, page, cell_bbox, source_text, confidence, flags)"
|
|
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
(new_uid(), scan_id, source["id"], e["date"], e.get("end_date") if e.get("end_date") != e["date"] else None,
|
|
e["all_day"], e.get("start_time"), e.get("end_time"), e["title"], e.get("notes"), e.get("location"),
|
|
e["kind"], e.get("applies", True), e.get("applies", True), page, json.dumps(e.get("cell_bbox")),
|
|
e.get("source_text"), e.get("confidence"), json.dumps(e.get("flags", [])))).lastrowid
|
|
conn.executemany("INSERT INTO event_child VALUES (?, ?)", [(event_id, c) for c in child_ids])
|
|
for m in r["menu_items"]:
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO menu_item (scan_id, source_id, date, items, no_lunch, page, cell_bbox,"
|
|
" confidence, flags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
(scan_id, source["id"], m["date"], json.dumps(m["items"]), m["no_lunch"], page,
|
|
json.dumps(m.get("cell_bbox")), m.get("confidence"), json.dumps(m.get("flags", []))))
|
|
dates = sorted(d for d in dates if d)
|
|
tokens = sum(r["usage"]["input_tokens"] + r["usage"]["output_tokens"] for r in runs)
|
|
conn.execute("UPDATE scan SET status = 'parsed', error = NULL, date_start = ?, date_end = ?, model = ?,"
|
|
" cost_tokens = ?, seconds = ? WHERE id = ?",
|
|
(dates[0] if dates else None, dates[-1] if dates else None, runs[0]["model"] if runs else None,
|
|
tokens, round(sum(r["seconds"] for r in runs), 1), scan_id))
|
|
match_drafts(conn, scan_id)
|
|
|
|
|
|
def overlaps(a, b):
|
|
return a["date"] <= (b["end_date"] or b["date"]) and b["date"] <= (a["end_date"] or a["date"])
|
|
|
|
|
|
def match_drafts(conn, scan_id):
|
|
"""Link each draft to the source's confirmed event it most likely is, so publishing updates that event
|
|
(same UID) instead of adding a duplicate. Same source, overlapping dates, similar title."""
|
|
drafts = db.all_dicts(conn.execute("SELECT * FROM event WHERE scan_id = ? AND status = 'draft'", (scan_id,)))
|
|
if not drafts:
|
|
return
|
|
existing = db.all_dicts(conn.execute(
|
|
"SELECT * FROM event WHERE source_id = ? AND status = 'confirmed' AND date <= ? AND COALESCE(end_date, date) >= ?",
|
|
(drafts[0]["source_id"], max(d["end_date"] or d["date"] for d in drafts), min(d["date"] for d in drafts))))
|
|
pairs = sorted(((SequenceMatcher(None, norm(d["title"]), norm(e["title"])).ratio(), d["id"], e["id"])
|
|
for d in drafts for e in existing if overlaps(d, e)), reverse=True)
|
|
by_id = {d["id"]: d for d in drafts} | {e["id"]: e for e in existing}
|
|
used = set()
|
|
for score, did, eid in pairs:
|
|
if score < 0.6 or did in used or eid in used:
|
|
continue
|
|
used |= {did, eid}
|
|
d, e = by_id[did], by_id[eid]
|
|
same = all(d[f] == e[f] for f in MATCH_FIELDS)
|
|
flags = d["flags"] + ([] if score >= SURE_MATCH else [f"might be the same as \"{e['title']}\" on {e['date']}"])
|
|
conn.execute("UPDATE event SET replaces_id = ?, match = ?, flags = ? WHERE id = ?",
|
|
(eid, "unchanged" if same else "updated", json.dumps(flags), did))
|
|
conn.execute("UPDATE event SET match = 'new' WHERE scan_id = ? AND status = 'draft' AND match IS NULL", (scan_id,))
|
|
|
|
|
|
def publish(conn, scan_id):
|
|
"""Checked drafts become confirmed events; a draft matched to an existing event updates it in place."""
|
|
# Until a scan is published, every event carrying its scan_id is one of its drafts
|
|
# (status 'deleted' here means deleted during review).
|
|
drafts = db.all_dicts(conn.execute("SELECT * FROM event WHERE scan_id = ?", (scan_id,)))
|
|
with conn:
|
|
for d in drafts:
|
|
if d["status"] == "deleted" or not d["include"]:
|
|
conn.execute("DELETE FROM event WHERE id = ?", (d["id"],))
|
|
elif d["replaces_id"]:
|
|
old = db.one(conn, "SELECT * FROM event WHERE id = ?", (d["replaces_id"],))
|
|
if any(d[f] != old[f] for f in MATCH_FIELDS) or old["status"] != "confirmed":
|
|
conn.execute(
|
|
"UPDATE event SET " + ", ".join(f"{f} = ?" for f in MATCH_FIELDS) +
|
|
", status = 'confirmed', sequence = sequence + 1, updated_at = strftime('%Y-%m-%d %H:%M:%f', 'now'),"
|
|
" scan_id = ?, page = ?, cell_bbox = ?, source_text = ? WHERE id = ?",
|
|
[d[f] for f in MATCH_FIELDS] + [scan_id, d["page"], json.dumps(d["cell_bbox"]),
|
|
d["source_text"], old["id"]])
|
|
conn.execute("DELETE FROM event_child WHERE event_id = ?", (old["id"],))
|
|
conn.execute("INSERT INTO event_child SELECT ?, child_id FROM event_child WHERE event_id = ?",
|
|
(old["id"], d["id"]))
|
|
conn.execute("DELETE FROM event WHERE id = ?", (d["id"],))
|
|
else:
|
|
conn.execute("UPDATE event SET status = 'confirmed', updated_at = strftime('%Y-%m-%d %H:%M:%f', 'now') WHERE id = ?", (d["id"],))
|
|
conn.execute("UPDATE scan SET status = 'published', published_at = CURRENT_TIMESTAMP WHERE id = ?", (scan_id,))
|
|
|
|
|
|
def clean_old_images(conn):
|
|
"""Delete page images IMAGE_RETENTION_DAYS after a scan was published or abandoned."""
|
|
cutoff = (datetime.now(timezone.utc) - timedelta(days=config.IMAGE_RETENTION_DAYS)).strftime("%Y-%m-%d %H:%M:%S")
|
|
old = db.all_dicts(conn.execute(
|
|
"SELECT id, image_keys FROM scan WHERE image_keys != '[]' AND ((status = 'published' AND published_at < ?)"
|
|
" OR (status = 'abandoned' AND created_at < ?))", (cutoff, cutoff)))
|
|
for scan in old:
|
|
for key in scan["image_keys"]:
|
|
for name in (key, processed_key(key)):
|
|
(config.IMAGES_DIR / name).unlink(missing_ok=True)
|
|
with conn:
|
|
conn.execute("UPDATE scan SET image_keys = '[]' WHERE id = ?", (scan["id"],))
|
|
|
|
|
|
def run_worker(stop):
|
|
"""Background thread: one job at a time, so the GPU only ever has one page in flight."""
|
|
with db.connect() as conn:
|
|
conn.execute("UPDATE job SET status = 'queued' WHERE status = 'running'") # interrupted by a restart
|
|
conn.commit()
|
|
last_cleanup = 0.0
|
|
while not stop.is_set():
|
|
conn = db.connect()
|
|
try:
|
|
if time.time() - last_cleanup > 3600:
|
|
clean_old_images(conn)
|
|
last_cleanup = time.time()
|
|
job = conn.execute("UPDATE job SET status = 'running', attempts = attempts + 1 WHERE id ="
|
|
" (SELECT id FROM job WHERE status = 'queued' ORDER BY id LIMIT 1) RETURNING id, scan_id"
|
|
).fetchone()
|
|
conn.commit()
|
|
if job is None:
|
|
stop.wait(2)
|
|
continue
|
|
try:
|
|
process_scan(conn, job["scan_id"])
|
|
status, error = "done", None
|
|
except Exception as e: # any failure marks this scan failed; the worker keeps serving the queue
|
|
log.exception("scan %s failed", job["scan_id"])
|
|
status, error = "failed", f"{type(e).__name__}: {e}"
|
|
with conn:
|
|
conn.execute("UPDATE job SET status = ?, error = ? WHERE id = ?", (status, error, job["id"]))
|
|
if error:
|
|
conn.execute("UPDATE scan SET status = 'failed', error = ? WHERE id = ?", (error, job["scan_id"]))
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def start_worker():
|
|
stop = threading.Event()
|
|
thread = threading.Thread(target=run_worker, args=(stop,), name="extraction-worker", daemon=True)
|
|
thread.start()
|
|
return stop, thread
|