diff --git a/parentcalendar/db.py b/parentcalendar/db.py index 5c14d79..dbe9416 100644 --- a/parentcalendar/db.py +++ b/parentcalendar/db.py @@ -7,7 +7,7 @@ from parentcalendar import config MIGRATIONS = Path(__file__).parent / "migrations" JSON_COLUMNS = {"source_ids", "ignore_labels", "image_keys", "items", "flags", "cell_bbox", "before", "after", - "child_ids"} # child_ids comes from json_group_array() subqueries + "child_ids", "reads"} # child_ids comes from json_group_array() subqueries BOOL_COLUMNS = {"double_read", "all_day", "applies", "include", "no_lunch"} diff --git a/parentcalendar/extraction.py b/parentcalendar/extraction.py index 0c67e73..b44727e 100644 --- a/parentcalendar/extraction.py +++ b/parentcalendar/extraction.py @@ -464,6 +464,7 @@ def merge_reads(a, b): m.setdefault("read_flags", []).append("second read did not find this") elif norm("".join(m["items"])) != norm("".join(o["items"])) or m["no_lunch"] != o["no_lunch"]: m.setdefault("read_flags", []).append("second read differs: items") + m["reads"] = [{"items": r["items"], "no_lunch": r["no_lunch"]} for r in (m, o)] # review can switch for o in menu_b.values(): o.setdefault("read_flags", []).append("only the second read found this") a["menu_items"].append(o) @@ -534,6 +535,7 @@ def selftest(): "No School": ["second read did not find this"], "Happy Halloween": ["only the second read found this"]}, flags assert m["menu_items"][0]["read_flags"] == ["second read differs: items"] + assert [r["items"] for r in m["menu_items"][0]["reads"]] == [["Corn"], ["Corn", "Peas"]] # A word in a stray alphabet is dropped; a page that is really bilingual keeps both page = {"events": [], "menu_items": [{"items": ["Chicken Patty", "Corn", "Whole Orange выда"]}, {"items": ["Pizza", "Garden Salad", "Mixed Fruit", "Peas", "Milk"]}]} diff --git a/parentcalendar/migrations/002_menu_reads.sql b/parentcalendar/migrations/002_menu_reads.sql new file mode 100644 index 0000000..0078af1 --- /dev/null +++ b/parentcalendar/migrations/002_menu_reads.sql @@ -0,0 +1,3 @@ +-- Both readings of a menu day when a double read disagreed: [{"items": [...], "no_lunch": bool}, {...}]. +-- NULL when the reads agreed or the source reads once. Review lets you switch between them. +ALTER TABLE menu_item ADD COLUMN reads TEXT; diff --git a/parentcalendar/scans.py b/parentcalendar/scans.py index 3eba721..d69b829 100644 --- a/parentcalendar/scans.py +++ b/parentcalendar/scans.py @@ -113,9 +113,10 @@ def process_scan(conn, scan_id): for m in r["menu_items"]: conn.execute( "INSERT OR REPLACE INTO menu_item (scan_id, source_id, date, items, no_lunch, page, cell_bbox," - " confidence, flags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + " confidence, flags, reads) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (scan_id, source["id"], m["date"], json.dumps(m["items"]), m["no_lunch"], page, - json.dumps(m.get("cell_bbox")), m.get("confidence"), json.dumps(m.get("flags", [])))) + json.dumps(m.get("cell_bbox")), m.get("confidence"), json.dumps(m.get("flags", [])), + json.dumps(m["reads"]) if m.get("reads") else None)) dates = sorted(d for d in dates if d) tokens = sum(r["usage"]["input_tokens"] + r["usage"]["output_tokens"] for r in runs) conn.execute("UPDATE scan SET status = 'parsed', error = NULL, date_start = ?, date_end = ?, model = ?,"