#!/usr/bin/env python3
"""Compare the 2019-2023 thresholds built here with PEER Center's published thresholds, cell by cell.

PEER Center, "The Fine Print: A Guide to Calculating Earnings Thresholds" (November 6, 2026),
https://www.peer-center.org/research/acs-earnings-thresholds; data file peer_thresholds_2019_2023.csv
from acs_thresholds_program.zip, CC BY 4.0. PEER keeps cents; this build keeps whole dollars, so a
difference of under one dollar is agreement.

Every cell gets a reason when the two differ:
  puerto_rico_in_national  PEER pools Puerto Rico into national bachelor's cells for 2019-2023;
                           this build follows the Department's high-school treatment and leaves it out
  zero_respondents         no ACS respondent in the state and field; both sides apply one dollar

Usage: python3 compare_thresholds_peer.py --date 20261106
Writes data/earnings-accountability/threshold_comparison_peer_<date>.csv and prints a summary.
"""
import argparse, csv, os
from collections import Counter

D = "data/earnings-accountability"
def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--date", required=True)
    ap.add_argument("--peer", default="_private/data/peer/peer_thresholds_2019_2023.csv")
    a = ap.parse_args()
    T = {r["state"]: r for r in csv.DictReader(open(f"{D}/earnings_thresholds_by_state_{a.date}_acs2023.csv"))}
    F = {(r["state"], r["cip2"]): r for r in csv.DictReader(open(f"{D}/earnings_thresholds_ba_by_state_field_{a.date}_acs2023.csv"))}
    PRD = {r["cip2"]: r for r in csv.DictReader(open(f"{D}/earnings_thresholds_ba_national_with_pr_diagnostic_{a.date}_acs2023.csv"))}
    ab = {r["state_fips"]: s for s, r in T.items() if r["state_fips"]}
    natba = int(T["US"]["ba_median_acs_positive_earnings"])
    natf = {c: int(r["ba_same_state_same_field_median"]) for (s, c), r in F.items() if s == "US"}
    out = []
    for r in csv.DictReader(open(a.peer)):
        lev, oos, fld, peer = r["level"], r["out_of_state"], r["field"], float(r["threshold"])
        st = "US" if r["state"] == "any" else ab[r["state"]]
        reason = ""
        if lev == "undergraduate":
            ours = int(T[st]["hs_median_acs_positive_earnings"])
        elif oos == "yes":
            cands = [natba] + ([natf[fld]] if fld in natf else [])
            ours = min(cands)
        elif fld == "other":
            ours = int(T[st]["ba_median_acs_positive_earnings"])
        else:
            c = F.get((st, fld))
            if c is None: ours = 1; reason = "zero_respondents"
            else: ours = 1 if c["below_16_respondents"] == "yes" else int(c["graduate_threshold_lowest_of_three"])
        diff = ours - peer
        if abs(diff) >= 1 and not reason and lev == "graduate":
            # Recompute with Puerto Rico in the national bachelor's cells, as PEER does
            nf_pr = int(PRD[fld]["ba_national_median_including_pr"]) if fld in PRD else None
            nb_pr = int(PRD["all"]["ba_national_median_including_pr"])
            if oos == "yes": alt = min([nb_pr] + ([nf_pr] if nf_pr else []))
            else:
                c = F[(st, fld)]
                alt = min(v for v in (int(c["ba_same_state_median"]), 1 if c["below_16_respondents"] == "yes" else int(c["ba_same_state_same_field_median"]), nf_pr) if v is not None)
            if abs(alt - peer) < 1: reason = "puerto_rico_in_national"
        out.append({"level": lev, "out_of_state": oos, "state": st, "field": fld, "threshold_opportunity_data": ours,
                    "threshold_peer": f"{peer:.2f}", "difference": f"{diff:.2f}", "agree_within_1_dollar": "yes" if abs(diff) < 1 else "no",
                    "peer_based_on": r["based_on"], "reason_for_difference": reason if abs(diff) >= 1 else ""})
    p = f"{D}/threshold_comparison_peer_{a.date}.csv"
    with open(p, "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=list(out[0].keys())); w.writeheader(); w.writerows(out)
    agree = sum(1 for r in out if r["agree_within_1_dollar"] == "yes")
    print("wrote", p, len(out), "rows; agree within $1:", agree)
    print(Counter((r["level"], r["agree_within_1_dollar"], r["reason_for_difference"]) for r in out))

if __name__ == "__main__":
    main()