54 lines
2.2 KiB
Python
54 lines
2.2 KiB
Python
"""Phase 0: run extraction on the golden set.
|
|
|
|
Usage: python phase0/extract.py [case ...] (default: every private/golden/*.json)
|
|
python phase0/extract.py --selftest
|
|
Writes private/runs/<timestamp>/<case>.json with the model output, validator flags and token usage.
|
|
Settings are the EXTRACT_* and OLLAMA_URL environment variables described in parentcalendar/extraction.py.
|
|
"""
|
|
import json
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from parentcalendar import extraction as x # noqa: E402
|
|
|
|
GOLDEN = ROOT / "private" / "golden"
|
|
RUNS = ROOT / "private" / "runs"
|
|
|
|
|
|
def main(names):
|
|
cases = [GOLDEN / f"{n}.json" for n in names] or sorted(GOLDEN.glob("*.json"))
|
|
out = RUNS / datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
out.mkdir(parents=True)
|
|
ollama = "" if x.MODEL.startswith("claude") else f" at {x.OLLAMA_URL}, think={x.OLLAMA_THINK}, temperature={x.TEMPERATURE}, top_p={x.TOP_P}, top_k={x.TOP_K}, num_ctx={x.NUM_CTX}, keep_alive={x.KEEP_ALIVE}"
|
|
print(f"Model: {x.MODEL}{ollama}")
|
|
tokens_in = tokens_out = 0
|
|
for path in cases:
|
|
print(f"{path.stem} ...", end=" ", flush=True)
|
|
case = json.loads(path.read_text(encoding="utf-8"))
|
|
try:
|
|
run = x.extract_image(x.load_image(ROOT / case["image"]), case.get("hint", {}))
|
|
except x.CALL_ERRORS as e:
|
|
print(f"FAILED: {e}")
|
|
continue
|
|
run.pop("image")
|
|
(out / path.name).write_text(json.dumps(run, indent=2), encoding="utf-8")
|
|
tokens_in += run["usage"]["input_tokens"]
|
|
tokens_out += run["usage"]["output_tokens"]
|
|
r = run["result"]
|
|
print(f"{len(r['events'])} events, {len(r['menu_items'])} menu days, {run['seconds']}s ({run['method']})")
|
|
print(f"\nTokens: {tokens_in} in, {tokens_out} out")
|
|
if x.MODEL == "claude-opus-5-5": # ponytail: price hardcoded ($4 in / $20 out per MTok)
|
|
print(f"About ${tokens_in * 4 / 1e6 + tokens_out * 20 / 1e6:.2f}")
|
|
print(f"Score it: python phase0/score.py {out.relative_to(ROOT)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if sys.argv[1:] == ["--selftest"]:
|
|
x.selftest()
|
|
else:
|
|
main(sys.argv[1:])
|