"""Hold-out scores (50 jobs never looked at during development), aggregates only - no per-job output.
Frozen system: rules v5 + Sonnet, text only (analysis/frozen_v5). Baseline: S0 engineer label + always 600x600 heavy.
Run once: python3 holdout_eval.py -> analysis/holdout_scores.json"""
import json, math, statistics as st
from collections import Counter
J = {j["item"]: j for j in json.load(open("jobs.json"))["jobs"]}
LAB = {e["item"]: e for e in json.load(open("labelled_examples.json"))["jobs"]}
rc = json.load(open("rate_card.json")); rc = rc["lines"] if isinstance(rc, dict) and "lines" in rc else rc
PR = {l["code"]: l["unit_price"] for l in rc}
HO = json.load(open("splits.json"))["holdout"]
R = json.load(open("analysis/quote_run_sonnet_holdout.json"))["records"]
MAIN = ("CVR-", "GUL-", "CHN-", "PIP-")
bad = [i for i in HO if "error" in R[i]["answer"] or "raw" in R[i]["answer"]]
assert not bad, f"{len(bad)} failed calls: run quote_run.py --holdout again first"
S5 = {i: dict(flag=bool(R[i]["answer"].get("is_manhole")), green=R[i]["answer"].get("colour") == "green",
              lines={l["code"]: l.get("qty", 1) for l in R[i]["answer"]["lines"] if l["code"] in PR}) for i in HO}
def s0(i):
    c = Counter(f.get("issue_type") for f in J[i]["remedial_forms"] if f.get("issue_type"))
    m = (c.most_common(1)[0][0] if c else None) == "Defective Manhole"
    return dict(flag=m, green=False, lines={"SRV-UTIL": 1, "WST-GEN": 1, "LAB-CIV-H": 1, **({"CVR-600-H": 1} if m else {})})
S0 = {i: s0(i) for i in HO}
def f1(tp, fp, fn):
    p = tp / (tp + fp) if tp + fp else 0; r = tp / (tp + fn) if tp + fn else 0
    return round(2 * p * r / (p + r) if p + r else 0, 2), round(p, 2), round(r, 2)
def score(P, ids):
    T = {i: {l["code"]: l["qty"] for l in LAB[i]["quote_lines"]} for i in ids}
    out = {"jobs": len(ids)}
    tp = sum(P[i]["flag"] and LAB[i]["is_manhole_job"] for i in ids); fp = sum(P[i]["flag"] and not LAB[i]["is_manhole_job"] for i in ids)
    fn = sum(not P[i]["flag"] and LAB[i]["is_manhole_job"] for i in ids)
    out["1 manhole F1 (P/R)"] = f1(tp, fp, fn)
    out["manhole yes/no right"] = sum(P[i]["flag"] == LAB[i]["is_manhole_job"] for i in ids)
    for name, m in (("2 main lines F1 (P/R)", True), ("3 other lines F1 (P/R)", False)):
        a = b = c = 0
        for i in ids:
            p = {x for x in P[i]["lines"] if x.startswith(MAIN) == m}; t = {x for x in T[i] if x.startswith(MAIN) == m}
            a += len(p & t); b += len(p - t); c += len(t - p)
        out[name] = f1(a, b, c)
    mh = [i for i in ids if LAB[i]["is_manhole_job"]]
    cov = lambda d: sorted(x for x in d if x.startswith("CVR-"))
    out["which cover right (manhole jobs)"] = f"{sum(P[i]['flag'] and cov(P[i]['lines']) == cov(T[i]) for i in mh)} of {len(mh)}"
    e = [sum(PR[c] * q for c, q in P[i]["lines"].items()) - LAB[i]["quote_total"] for i in ids]
    out["4 MAE £"] = round(st.mean(abs(x) for x in e)); out["4 RMSE £"] = round(math.sqrt(st.mean(x * x for x in e)))
    out["median abs error £"] = round(st.median(abs(x) for x in e))
    out["total within 10%"] = sum(abs(x) <= 0.1 * LAB[i]["quote_total"] for x, i in zip(e, ids))
    out["total within 25%"] = sum(abs(x) <= 0.25 * LAB[i]["quote_total"] for x, i in zip(e, ids))
    out["main lines exactly right"] = sum({x for x in P[i]["lines"] if x.startswith(MAIN)} == {x for x in T[i] if x.startswith(MAIN)} for i in ids)
    return out
res = {"S0 engineer label": score(S0, HO), "S5 frozen (rules v5 + Sonnet)": score(S5, HO)}
g = [i for i in HO if S5[i]["green"]]
if g: res["S5 green only"] = score(S5, g)
json.dump(res, open("analysis/holdout_scores.json", "w"), indent=1)
for k, v in res.items():
    print(f"\n{k}"); [print(f"   {a}: {b}") for a, b in v.items()]