"""Grade one slate: every source's projections, ownership and lineups against what happened.""" from __future__ import annotations import csv import json import math import re from collections import Counter from pathlib import Path import numpy as np import pandas as pd from scipy.stats import spearmanr from .adapters import load_projections from .baselines import append_history, load_history, recent_form_baseline, salary_baseline from .capture import LOG, RESULTS_META, SALARY_FILE, verify_fingerprints from .config import is_graded, load_aliases, load_sources, sport_cfg from .dk import lineup_key, parse_answer_rows, parse_lineup_string, read_lineup_file, read_salaries, read_standings from .names import PlayerIndex from .solver import best_lineup, check_lineup BASELINES = ["baseline_recent", "baseline_salary"] TOP_PCT = 0.99 # "top 1%" finish def clean(obj): """JSON-safe: numpy -> python, NaN -> None.""" if isinstance(obj, dict): return {str(k): clean(v) for k, v in obj.items()} if isinstance(obj, (list, tuple, set)): return [clean(v) for v in obj] if isinstance(obj, (np.integer,)): return int(obj) if isinstance(obj, (np.floating, float)): v = float(obj) return None if math.isnan(v) or math.isinf(v) else round(v, 6) if isinstance(obj, np.bool_): return bool(obj) return obj def slate_date(slate_dir: Path) -> str: m = re.match(r"(\d{4}-\d{2}-\d{2})", slate_dir.name) if m: return m.group(1) raise ValueError(f"{slate_dir}: folder name should start with the date, like 2026-10-20-main") def _read_json(path: Path) -> dict: return json.loads(path.read_text()) if path.exists() else {} def _id_list(path: Path, index: PlayerIndex) -> set[str]: if not path.exists(): return set() ids = set() with path.open(newline="", encoding="utf-8-sig") as f: for row in csv.DictReader(f): row = {(k or "").strip().lower(): (v if isinstance(v, str) else "").strip() for k, v in row.items()} pid = index.match(name=row.get("name") or row.get("player"), team=row.get("team"), dk_id=row.get("dk_id") or None) if pid: ids.add(pid) return ids def _cash_line(res: Path, meta: dict, warnings: list) -> float | None: path = res / "doubleup_standings.csv" if not path.exists(): warnings.append("No double-up standings: Cash-Line Rate skipped for this slate") return None entries, _ = read_standings(path) if entries.empty: return None paid = meta.get("doubleup_paid_places") if not paid: paid = int(len(entries) * 0.444) warnings.append(f"results.json has no doubleup_paid_places; assumed {paid} of {len(entries)} paid") ranked = entries.sort_values("rank") return float(ranked["points"].iloc[: int(paid)].min()) def _payout_table(meta: dict) -> list[tuple[int, int, float]]: """results.json gpp_payouts: [{"from": 1, "to": 1, "prize": 1000}, ...] or [[1, 1, 1000], ...].""" out = [] for p in meta.get("gpp_payouts") or []: if isinstance(p, dict): out.append((int(p["from"]), int(p["to"]), float(p["prize"]))) elif isinstance(p, (list, tuple)) and len(p) == 3: out.append((int(p[0]), int(p[1]), float(p[2]))) return out def grade_slate(slate_dir: Path, sport: str, write: bool = True) -> dict: slate_dir = Path(slate_dir) cfg = sport_cfg(sport) cap, res = slate_dir / "capture", slate_dir / "results" warnings: list[str] = [] date = slate_date(slate_dir) for need, what in ((cap / SALARY_FILE, "the DraftKings salary file"), (res / "gpp_standings.csv", "the reference GPP's standings")): if not need.exists(): raise FileNotFoundError(f"{slate_dir.name}: missing {what} ({need.relative_to(slate_dir)})") meta = _read_json(res / RESULTS_META) log = _read_json(cap / LOG) warnings += [f"Fingerprints: {w}" for w in verify_fingerprints(slate_dir, sport)] sal = read_salaries(cap / SALARY_FILE, sport) sal_idx = sal.set_index("dk_id") index = PlayerIndex(sal, sport, load_aliases()) sources = {} for s in load_sources(): if is_graded(s, sport): sources[s["id"]] = s elif any((cap / f).exists() for f in s.get("files", {}).values()): warnings.append(f"{s['name']}: files present but participation is '{s.get('participation')}'; not graded") # 1. Projections from the tools and the house entry. proj: dict[str, pd.DataFrame] = {} reports, detected = {}, {} for sid, s in sources.items(): fname = s["files"].get("projections") if s["kind"] in ("tool", "house") and fname and (cap / fname).exists(): try: df, rep, cols = load_projections(cap / fname, s, index) except ValueError as e: warnings.append(str(e)) continue proj[sid], reports[sid], detected[sid] = df, rep.as_dict(), cols if rep.unmatched: warnings.append(f"{s['name']}: {len(rep.unmatched)} rows not matched to DraftKings players (see match report)") tool_ids = list(proj) # 2. Who was ruled out at the capture minute: the tools' consensus plus an optional manual list. ruled_out: set[str] = set() if tool_ids: tm = pd.DataFrame({sid: proj[sid]["proj"] for sid in tool_ids}).reindex(sal["dk_id"]) med = tm.fillna(0.0).median(axis=1) avg = sal_idx["avg_ppg"].reindex(med.index) ruled_out |= set(med[(med <= 0.5) & (avg >= cfg["pool_floor"] / 2)].index) ruled_out |= _id_list(cap / "ruled_out.csv", index) # 3. Baselines (capture-time information only; ruled-out players get 0). hist_path = slate_dir.parent / "_history.csv" history = load_history(hist_path) b_recent, used = recent_form_baseline(sal, history, cfg["recent_games"], date) b_salary = salary_baseline(sal) for bid, b in (("baseline_recent", b_recent), ("baseline_salary", b_salary)): b = b.where(~b.index.isin(ruled_out), 0.0) proj[bid] = pd.DataFrame({"proj": b, "own": np.nan, "ceil": np.nan, "floor": np.nan}) projection_ids = tool_ids + BASELINES # 4. What actually happened: points and ownership from the reference GPP's standings. entries, players = read_standings(res / "gpp_standings.csv") actual, own_actual, unmatched_actual = {}, {}, [] for rec in players.to_dict("records"): pid = index.match(name=rec["name"], pos=rec["roster_pos"] or None) if pid: actual[pid] = rec["fpts"] own_actual[pid] = rec["drafted_pct"] else: unmatched_actual.append(rec["name"]) if unmatched_actual: warnings.append(f"{len(unmatched_actual)} standings players not matched: {', '.join(unmatched_actual[:5])}") ov = res / "actuals_override.csv" if ov.exists(): with ov.open(newline="", encoding="utf-8-sig") as f: for row in csv.DictReader(f): row = {(k or "").strip().lower(): (v if isinstance(v, str) else "").strip() for k, v in row.items()} pid = index.match(name=row.get("name"), team=row.get("team"), dk_id=row.get("dk_id") or None) if pid and row.get("fpts") not in (None, ""): actual[pid] = float(row["fpts"]) elif row.get("name") or row.get("dk_id"): warnings.append(f"actuals_override.csv: couldn't match '{row.get('name') or row.get('dk_id')}'") actual_full = pd.Series(actual, dtype=float).reindex(sal["dk_id"]) actual_full[actual_full.index.isin(ruled_out) & actual_full.isna()] = 0.0 own_full = pd.Series(own_actual, dtype=float).reindex(sal["dk_id"]).fillna(0.0) # 5. Late scratches: listed by hand, plus (NBA) projected rotation players who scored exactly zero. consensus_src = tool_ids if tool_ids else BASELINES consensus = pd.DataFrame({sid: proj[sid]["proj"] for sid in consensus_src}).reindex(sal["dk_id"]).median(axis=1) late = _id_list(res / "late_scratches.csv", index) if cfg.get("zero_points_means_scratch"): z = actual_full[(actual_full == 0) & (consensus >= cfg["pool_floor"]) & ~actual_full.index.isin(ruled_out)] late |= set(z.index) late -= ruled_out actual_full[actual_full.index.isin(late)] = 0.0 # 6. The graded pool: every projection source projected them, consensus clears the floor, # and we know what they scored. Ruled-out players stay in when anyone projected them # for real points, so a stale projection counts as the miss it is. allp = pd.DataFrame({sid: proj[sid]["proj"] for sid in projection_ids}).reindex(sal["dk_id"]) is_out = allp.index.isin(ruled_out) allp.loc[is_out] = allp.loc[is_out].fillna(0.0) known = actual_full.notna().to_numpy() active_ok = (~is_out) & (consensus >= cfg["pool_floor"]).to_numpy() & allp.notna().all(axis=1).to_numpy() stale_ok = is_out & (allp.max(axis=1) >= cfg["pool_floor"]).to_numpy() mask = known & ~allp.index.isin(late) & (active_ok | stale_ok) pool = allp[mask] act = actual_full.reindex(pool.index) group = sal_idx["pos"].reindex(pool.index).map(cfg["position_groups"]) pos = sal_idx["pos"].reindex(pool.index) salk = sal_idx["salary"].reindex(pool.index) / 1000.0 if len(pool) < 20: warnings.append(f"Only {len(pool)} players in the graded pool; check that every source covers this slate") # 7. The perfect lineup, the cash line and the field. opt = best_lineup(sal, actual_full.dropna(), cfg) or [] opt_ids = [pid for _, pid in opt] opt_pts = float(actual_full.reindex(opt_ids).sum()) if opt_ids else float("nan") cash_line = _cash_line(res, meta, warnings) field_pts = entries["points"].to_numpy(dtype=float) if len(entries) else np.array([]) field_keys = Counter( lineup_key([n for _, n in parse_lineup_string(s, cfg["slot_tokens"])]) for s in entries.get("lineup", []) ) payouts = _payout_table(meta) fee = meta.get("gpp_entry_fee") def lineup_record(ids: list[str]) -> dict: pts = float(actual_full.reindex(ids).fillna(0.0).sum()) rec = { "dk_ids": ids, "points": pts, "pct_perfect": pts / opt_pts if opt_pts else None, "cashed": (pts >= cash_line) if cash_line is not None else None, "overlap": len(set(ids) & set(opt_ids)), "donuts": int((actual_full.reindex(ids).fillna(0.0) <= cfg["donut_max"]).sum()), "salary": int(sal_idx["salary"].reindex(ids).sum()), } if len(field_pts): less = float((field_pts < pts).sum()) eq = float((field_pts == pts).sum()) rec["field_percentile"] = (less + 0.5 * eq) / len(field_pts) rec["top1"] = rec["field_percentile"] >= TOP_PCT rec["finish_rank"] = int((field_pts > pts).sum()) + 1 rec["dupes"] = field_keys.get(lineup_key(sal_idx["name"].reindex(ids).tolist()), 0) if payouts and fee: rec["entry_fee"] = float(fee) rec["winnings"] = next((prize for lo, hi, prize in payouts if lo <= rec["finish_rank"] <= hi), 0.0) return rec # 8. Per-source metrics. base_err = (pool["baseline_recent"] - act).abs() metrics: dict[str, dict] = {} for sid in projection_ids: p = pool[sid] err = (p - act).abs() m = { "kind": sources[sid]["kind"], "n": int(len(p)), "sum_abs": float(err.sum()), "sum_abs_base": float(base_err.sum()), "mae": float(err.mean()) if len(p) else None, "btb": 1.0 - float(err.sum()) / float(base_err.sum()) if base_err.sum() > 0 else None, "groups": { g: {"n": int((group == g).sum()), "sum_abs": float(err[group == g].sum()), "mae": float(err[group == g].mean())} for g in sorted(group.dropna().unique()) }, } rc, wn = [], [] for _, idx in p.groupby(pos).groups.items(): if len(idx) >= 5 and p.loc[idx].nunique() > 1 and act.loc[idx].nunique() > 1: r = spearmanr(p.loc[idx], act.loc[idx]).statistic if not np.isnan(r): rc.append(float(r)) wn.append(len(idx)) m["rank"] = float(np.average(rc, weights=wn)) if rc else None m["rank_n"] = int(sum(wn)) if len(p) >= cfg["value_top_projected"]: top_actual = (act / salk).nlargest(cfg["value_top_actual"]).index top_proj = (p / salk).nlargest(cfg["value_top_projected"]).index m["value"] = len(set(top_actual) & set(top_proj)) / len(top_actual) if sid in tool_ids and proj[sid]["own"].notna().sum() >= 5: own_p = proj[sid]["own"].reindex(pool.index) o_err = (own_p - own_full.reindex(pool.index)).abs().dropna() m["own_n"] = int(len(o_err)) m["own_sum_abs"] = float(o_err.sum()) m["own_mae"] = float(o_err.mean()) if len(o_err) else None k = cfg["chalk_top"] m["chalk"] = len(set(own_full.nlargest(k).index) & set(proj[sid]["own"].dropna().nlargest(k).index)) / k vals = proj[sid]["proj"].reindex(sal["dk_id"]) lu = best_lineup(sal, vals, cfg) if lu: m["same_engine"] = lineup_record([pid for _, pid in lu]) else: warnings.append(f"{sid}: no legal same-engine lineup") metrics[sid] = m # 8b. Reported, never ranked (rulebook section 8). All from graded errors; nothing here is a projection. # Tendencies: signed error (projected minus actual) by position group and salary tier, and ownership's. tiers = cfg.get("salary_tiers") or [] sal_k = sal_idx["salary"].reindex(pool.index) tier = pd.Series([next((lab for lo, hi, lab in tiers if lo <= x < hi), None) for x in sal_k], index=pool.index) signed = pool[projection_ids].sub(act, axis=0) for sid in projection_ids: e = signed[sid] lean = { "all": [int(len(e)), float(e.sum())], "groups": {g: [int((group == g).sum()), float(e[group == g].sum())] for g in sorted(group.dropna().unique())}, "tiers": {lab: [int((tier == lab).sum()), float(e[tier == lab].sum())] for _, _, lab in tiers if (tier == lab).any()}, } if sid in tool_ids and proj[sid]["own"].notna().sum() >= 5: oe = (proj[sid]["own"].reindex(pool.index) - own_full.reindex(pool.index)).dropna() chalk_ids = own_full.reindex(pool.index).nlargest(cfg["chalk_top"] * 2).index oc = oe.reindex(chalk_ids).dropna() lean["own"] = [int(len(oe)), float(oe.sum())] lean["own_chalk"] = [int(len(oc)), float(oc.sum())] metrics[sid]["lean"] = lean # Error overlap: sums for a pooled correlation of every pair of sources' errors. overlap = {} for i, a in enumerate(projection_ids): x = signed[a].to_numpy(dtype=float) for b in projection_ids[i + 1:]: y = signed[b].to_numpy(dtype=float) overlap[f"{a}|{b}"] = [int(len(x)), float(x.sum()), float(y.sum()), float((x * x).sum()), float((y * y).sum()), float((x * y).sum())] # Industry misses: players every graded source, the baselines included, missed by the floor or more. miss_floor = float(cfg.get("miss_floor", 12)) absd = signed.abs() industry_misses = [] for pid in pool.index: low = float(absd.loc[pid].min()) if low < miss_floor: continue e = signed.loc[pid] industry_misses.append({ "dk_id": pid, "name": sal_idx.at[pid, "name"], "team": sal_idx.at[pid, "team"], "pos": sal_idx.at[pid, "pos"], "salary": int(sal_idx.at[pid, "salary"]), "actual": float(act[pid]), "min_miss": low, "direction": "under" if (e < 0).all() else ("over" if (e > 0).all() else "split"), "closest": sorted(absd.loc[pid][absd.loc[pid] <= low + 1e-9].index.tolist()), }) industry_misses.sort(key=lambda m: -m["min_miss"]) # 9. Each tool's own #1 lineup, and Claude's. lineups: dict[str, dict] = {} answer_slots: dict[str, list] = {} need = sum(cfg["slots"].values()) for sid, s in sources.items(): if s["kind"] not in ("tool", "house", "ai"): continue lf, af = s["files"].get("lineup"), s["files"].get("answer") if lf and (cap / lf).exists(): ids, warns = read_lineup_file(cap / lf, index, cfg["slot_tokens"]) elif af and (cap / af).exists(): pairs, warns = parse_answer_rows((cap / af).read_text(encoding="utf-8", errors="replace"), index) ids = [pid for _, pid in pairs] answer_slots[sid] = pairs else: continue warnings += warns rec = {"kind": s["kind"]} chk = check_lineup(ids, sal, cfg) rec.update(chk) rec["dk_ids"] = ids rec["stale"] = bool(set(ids) & ruled_out) if len(ids) == need and chk["valid"]: rec.update(lineup_record(ids)) elif s["kind"] != "ai": warnings.append(f"{s['name']}: #1 lineup isn't a legal lineup ({'; '.join(chk['problems'])}); not graded") lineups[sid] = rec def lineup_rows(pairs): return [ { "slot": slot, "dk_id": pid, "name": sal_idx.at[pid, "name"], "team": sal_idx.at[pid, "team"], "salary": int(sal_idx.at[pid, "salary"]), "fpts": float(actual_full[pid]) if not pd.isna(actual_full[pid]) else None, "own": float(own_full[pid]), } for slot, pid in pairs ] ai_lineups = {} for sid, rec in lineups.items(): if rec["kind"] == "ai": # Shown as returned, with the slots Claude wrote. pairs = [(slot or sal_idx.at[pid, "pos"], pid) for slot, pid in answer_slots.get(sid, []) if pid in sal_idx.index] ai_lineups[sid] = lineup_rows(pairs) graded = { "slate": slate_dir.name, "sport": sport, "date": date, "players_on_slate": int(len(sal)), "games": int(sal["game"].nunique()), "pool_size": int(len(pool)), # Listed publicly: ruled-out players who mattered (DK average at or above the pool floor). "ruled_out": sorted(sal_idx.loc[[p for p in ruled_out if sal_idx.at[p, "avg_ppg"] >= cfg["pool_floor"]], "name"].tolist()), "ruled_out_total": len(ruled_out), "late_scratches": sorted(sal_idx["name"].reindex(sorted(late)).dropna().tolist()), "baseline_rule_counts": used, "cash_line": cash_line, "field_size": int(len(entries)), "reference": {k: meta.get(k) for k in ("gpp_contest", "doubleup_contest", "gpp_entry_fee")}, "capture": { "claude_model": log.get("sources", {}).get("claude", {}).get("model") or None, "plans": {sid: v.get("plan") for sid, v in log.get("sources", {}).items() if v.get("plan")}, }, "optimal": { "points": opt_pts, "salary": int(sal_idx["salary"].reindex(opt_ids).sum()) if opt_ids else None, "lineup": lineup_rows(opt), }, "sources": metrics, "overlap": overlap, "miss_floor": miss_floor, "industry_misses": industry_misses, "lineups": lineups, "ai_lineups": ai_lineups, "match_reports": reports, "detected_columns": detected, "warnings": warnings, } graded = clean(graded) if write: out = slate_dir / "graded" out.mkdir(exist_ok=True) (out / "graded.json").write_text(json.dumps(graded, indent=2)) rows = pd.DataFrame({ "dk_id": pool.index, "name": sal_idx["name"].reindex(pool.index).to_numpy(), "team": sal_idx["team"].reindex(pool.index).to_numpy(), "pos": pos.to_numpy(), "salary": sal_idx["salary"].reindex(pool.index).to_numpy(), "actual": act.to_numpy(), "actual_own": own_full.reindex(pool.index).to_numpy(), "ruled_out": pool.index.isin(ruled_out), }) for sid in projection_ids: rows[f"proj_{sid}"] = pool[sid].to_numpy() if sid in tool_ids: rows[f"own_{sid}"] = proj[sid]["own"].reindex(pool.index).to_numpy() rows.to_csv(out / "players.csv", index=False) # PRIVATE: contains the tools' projections # Public-safe: DFS Benchmark's own baselines for every player on the slate. pd.DataFrame({ "dk_id": sal["dk_id"], "name": sal["name"], "team": sal["team"], "pos": sal["pos"], "salary": sal["salary"], "ruled_out_at_capture": sal["dk_id"].isin(ruled_out), "baseline_recent": proj["baseline_recent"]["proj"].reindex(sal["dk_id"]).round(2).to_numpy(), "baseline_salary": proj["baseline_salary"]["proj"].reindex(sal["dk_id"]).round(2).to_numpy(), "actual": actual_full.reindex(sal["dk_id"]).to_numpy(), }).to_csv(out / "baselines.csv", index=False) played = actual_full.dropna() played = played[~played.index.isin(late) & ~played.index.isin(ruled_out)] if cfg.get("zero_points_means_scratch"): played = played[played != 0] # NBA: exactly zero almost always means he didn't play append_history(hist_path, pd.DataFrame({ "date": date, "name_key": sal_idx["name_key"].reindex(played.index).to_numpy(), "team": sal_idx["team"].reindex(played.index).to_numpy(), "fpts": played.to_numpy(), })) return graded