"""Pool graded slates into season numbers with 95% ranges, and decide who leads. Rules (rulebook section 8): - Every number carries a 95% range from resampling slates (bootstrap). - A tool leads a row only when its range clears the next tool's; otherwise the row is tied. - No row is ranked before the sport's minimum number of slates. - Tools and the house entry are ranked. Baselines are shown for reference, never ranked. Claude is graded in its own AI Division. - Reported, never ranked: tendencies (average signed error by position group and salary tier), shared mistakes between sources (independence), and industry misses. All come from graded errors, never from publishing a projection. """ from __future__ import annotations import json from itertools import combinations from pathlib import Path import numpy as np from .config import cost_per_slate, load_sources, season_of, slates_dir, sport_cfg from .grade import clean N_BOOT = 2000 SEED = 20261020 # fixed so every published range can be reproduced exactly RANKED_KINDS = ("tool", "house") HEADLINE = [ {"id": "btb", "name": "Beat-the-Baseline", "better": "higher", "unit": "pct", "pool": ("ratio", "sum_abs", "sum_abs_base"), "one_minus": True}, {"id": "mae", "name": "Points Accuracy", "better": "lower", "unit": "pts", "pool": ("ratio", "sum_abs", "n")}, {"id": "rank", "name": "Rank Accuracy", "better": "higher", "unit": "corr", "pool": ("wmean", "rank", "rank_n")}, {"id": "value", "name": "Value Finder", "better": "higher", "unit": "pct", "pool": ("mean", "value")}, {"id": "own_mae", "name": "Ownership Accuracy", "better": "lower", "unit": "own_pts", "pool": ("ratio", "own_sum_abs", "own_n")}, {"id": "chalk", "name": "Chalk Finder", "better": "higher", "unit": "pct", "pool": ("mean", "chalk")}, {"id": "pct_perfect", "name": "% of Perfect", "better": "higher", "unit": "pct", "pool": ("mean", "same_engine.pct_perfect")}, {"id": "cash_rate", "name": "Cash-Line Rate", "better": "higher", "unit": "pct", "pool": ("mean", "same_engine.cashed")}, ] AWARDS = [ {"id": "most_accurate", "name": "Most Accurate Projections", "benchmark": "btb"}, {"id": "best_ownership", "name": "Best Ownership", "benchmark": "own_mae"}, {"id": "best_value_finder", "name": "Best Value Finder", "benchmark": "value"}, {"id": "best_value_for_money", "name": "Best Value for Money", "benchmark": "btb", "rule": "cheapest graded plan among the tools tied for the best Beat-the-Baseline"}, {"id": "most_improved", "name": "Most Improved", "benchmark": "form", "rule": "largest rise in Beat-the-Baseline from the first to the last form window"}, ] # --------------------------------------------------------------------------- loading def load_graded(sport: str, season: str | None = None) -> list[dict]: out = [] for path in sorted(slates_dir(sport).glob("*/graded/graded.json")): g = json.loads(path.read_text()) if season and season_of(g["date"], sport) != season: continue out.append(g) out.sort(key=lambda g: (g["date"], g["slate"])) return out def latest_season(sport: str) -> str | None: graded = load_graded(sport) return season_of(graded[-1]["date"], sport) if graded else None def _get(d, path: str): for k in path.split("."): if not isinstance(d, dict) or k not in d or d[k] is None: return None d = d[k] return d # --------------------------------------------------------------------------- pooling def _weights(n_slates: int, seed) -> np.ndarray: """Bootstrap resample counts, shape (N_BOOT, n_slates). The same draws are used for every source (paired).""" if n_slates == 0: return np.zeros((N_BOOT, 0)) rng = np.random.default_rng(seed) idx = rng.integers(0, n_slates, size=(N_BOOT, n_slates)) return np.stack([np.bincount(r, minlength=n_slates) for r in idx]).astype(float) def _pool(W: np.ndarray, X: np.ndarray, D: np.ndarray, one_minus: bool = False) -> list[dict]: """Pooled value and 95% range per column. X/D are (slates, sources); value = sum X / sum D.""" out = [] with np.errstate(invalid="ignore", divide="ignore"): tot_x, tot_d = X.sum(0), D.sum(0) val = np.where(tot_d > 0, tot_x / np.where(tot_d > 0, tot_d, 1), np.nan) boot = (W @ X) / (W @ D) if len(W) else np.full((1, X.shape[1]), np.nan) if one_minus: val, boot = 1 - val, 1 - boot for k in range(X.shape[1]): n = int((D[:, k] > 0).sum()) col = boot[:, k][np.isfinite(boot[:, k])] if n == 0 or np.isnan(val[k]): out.append({"value": None, "lo": None, "hi": None, "n_slates": n}) continue lo, hi = (np.percentile(col, [2.5, 97.5]) if len(col) else (np.nan, np.nan)) out.append({"value": float(val[k]), "lo": float(lo), "hi": float(hi), "n_slates": n}) return out def _matrix(graded: list[dict], sids: list[str], spec: tuple, where: str = "sources", fill=None): """Build (X, D) for a pooling spec. fill(record, slate) can supply a value for records missing it.""" S, K = len(graded), len(sids) X, D = np.zeros((S, K)), np.zeros((S, K)) kind = spec[0] for s, g in enumerate(graded): for k, sid in enumerate(sids): m = g.get(where, {}).get(sid) if m is None: continue if kind == "ratio": num, den = _get(m, spec[1]), _get(m, spec[2]) if num is None or not den: continue X[s, k], D[s, k] = float(num), float(den) elif kind == "wmean": x, w = _get(m, spec[1]), _get(m, spec[2]) if x is None or not w: continue X[s, k], D[s, k] = float(x) * w, float(w) elif kind == "mean": x = _get(m, spec[1]) if x is None and fill is not None: x = fill(m, g) if x is None: continue X[s, k], D[s, k] = float(x), 1.0 return X, D def _status(rows: list[dict], better: str, min_slates: int, any_ranked: bool) -> dict: """Rank eligible rows; a leader only when its range clears the next one's.""" for r in rows: r["rank"] = None if not any_ranked: return {"status": "provisional", "leader": None, "tied": []} elig = [r for r in rows if r["ranked"] and r["value"] is not None and r["n_slates"] >= min_slates] sign = 1 if better == "higher" else -1 elig.sort(key=lambda r: -sign * r["value"]) for i, r in enumerate(elig): r["rank"] = i + 1 if not elig: return {"status": "provisional", "leader": None, "tied": []} if len(elig) == 1: return {"status": "no_contest", "leader": None, "tied": [elig[0]["source"]]} top, nxt = elig[0], elig[1] clear = top["lo"] > nxt["hi"] if better == "higher" else top["hi"] < nxt["lo"] if clear: return {"status": "leader", "leader": top["source"], "tied": []} tied = [r["source"] for r in elig if not (r["hi"] < top["lo"] or r["lo"] > top["hi"])] return {"status": "tied", "leader": None, "tied": tied} # --------------------------------------------------------------------------- the season def aggregate(sport: str, season: str | None = None) -> dict: cfg = sport_cfg(sport) season = season or latest_season(sport) graded = load_graded(sport, season) if season else [] sources = [s for s in load_sources()] by_id = {s["id"]: s for s in sources} proj_ids = [s["id"] for s in sources if s["kind"] in RANKED_KINDS + ("baseline",)] lineup_ids = [s["id"] for s in sources if s["kind"] in RANKED_KINDS + ("ai",)] ai_ids = [s["id"] for s in sources if s["kind"] == "ai"] min_slates = cfg["min_slates_to_rank"] S = len(graded) W = _weights(S, SEED) any_ranked = S >= min_slates def row(sid, stats): s = by_id[sid] return {"source": sid, "name": s["name"], "kind": s["kind"], "ranked": s["kind"] in RANKED_KINDS, **stats} # Headline rows. benchmarks = [] for b in HEADLINE: X, D = _matrix(graded, proj_ids, b["pool"]) pooled = _pool(W, X, D, b.get("one_minus", False)) rows = [row(sid, st) for sid, st in zip(proj_ids, pooled) if st["n_slates"] > 0] status = _status(rows, b["better"], min_slates, any_ranked) sign = 1 if b["better"] == "higher" else -1 rows.sort(key=lambda r: (r["rank"] is None, r["rank"] or 0, -sign * r["value"] if r["value"] is not None else np.inf)) entry = {k: b[k] for k in ("id", "name", "better", "unit")} entry.update(status) entry["rows"] = rows if b["id"] == "mae": entry["by_group"] = _groups(graded, proj_ids) benchmarks.append(entry) bench = {b["id"]: b for b in benchmarks} # Record rows: own #1 lineups (tools, house and Claude) and the same-engine lineups. record = { "own_lineup": _lineup_records(graded, lineup_ids, by_id, W, where="lineups"), "same_engine_overlap": [ row(sid, st) for sid, st in zip(proj_ids, _pool(W, *_matrix(graded, proj_ids, ("mean", "same_engine.overlap")))) if st["n_slates"] > 0 ], } # AI Division. ai = {} for sid in ai_ids: answered = [g for g in graded if sid in g.get("lineups", {})] recs = _lineup_records(graded, [sid], by_id, W, where="lineups", ai=True) if not answered or not recs: continue ai[sid] = { "name": by_id[sid]["name"], "slates": len(answered), "models": sorted({g["capture"]["claude_model"] for g in answered if _get(g, "capture.claude_model")}), "metrics": {r["metric"]: {k: r[k] for k in ("value", "lo", "hi", "n_slates")} for r in recs[0]["metrics"]}, } # Cost and accuracy vs cost. cost = [] for sid in proj_ids: s = by_id[sid] if s["kind"] not in RANKED_KINDS: continue plans = cost_per_slate(s, sport) graded_plan = next((p for p in plans if p["graded"]), None) cheapest = min((p for p in plans if p["per_slate"] is not None), key=lambda p: p["per_slate"], default=None) btb_row = next((r for r in bench["btb"]["rows"] if r["source"] == sid), None) cost.append({ "source": sid, "name": s["name"], "kind": s["kind"], "plans": plans, "graded_plan": graded_plan["name"] if graded_plan else None, "graded_per_slate": graded_plan["per_slate"] if graded_plan else None, "cheapest_per_slate": cheapest["per_slate"] if cheapest else None, "btb": {k: btb_row[k] for k in ("value", "lo", "hi", "n_slates")} if btb_row else None, }) form = _form(graded, [sid for sid in proj_ids if by_id[sid]["kind"] in RANKED_KINDS or sid == "baseline_salary"], by_id, cfg["form_window"]) awards = _awards(bench, cost, form, cfg, any_ranked) badges = [] for b in benchmarks: if b["status"] == "leader": s = by_id[b["leader"]] badges.append({ "source": s["id"], "name": s["name"], "house": s["kind"] == "house", "benchmark": b["id"], "label": f"#1 {b['name']}, {cfg['label']} {season.replace('-', '–')}" if season else None, }) return clean({ "sport": sport, "label": cfg["label"], "season": season, "n_slates": S, "first_date": graded[0]["date"] if graded else None, "last_date": graded[-1]["date"] if graded else None, "min_slates_to_rank": min_slates, "ranked": any_ranked, "bootstrap": {"resamples": N_BOOT, "seed": SEED, "range": "95%"}, "sources": [ {"id": s["id"], "name": s["name"], "kind": s["kind"], "color_slot": s.get("color_slot"), "label": "House" if s["kind"] == "house" else ("AI" if s["kind"] == "ai" else ("Baseline" if s["kind"] == "baseline" else None))} for s in sources ], "benchmarks": benchmarks, "record": record, "ai_division": ai, "cost": cost, "form": form, "awards": awards, "badges": badges, "tendencies": _lean(graded, proj_ids, by_id, W, cfg), "overlap": _overlap(graded, proj_ids, by_id, W), "misses": _misses(graded, proj_ids, by_id, cfg), "slates": [{"slate": g["slate"], "date": g["date"]} for g in graded], }) def _groups(graded: list[dict], sids: list[str]) -> list[dict]: """Points Accuracy by position group, pooled over the season.""" tot: dict = {} for g in graded: for sid in sids: for grp, v in (g["sources"].get(sid, {}).get("groups") or {}).items(): t = tot.setdefault((sid, grp), [0.0, 0]) t[0] += v["sum_abs"] t[1] += v["n"] out = [] for (sid, grp), (sa, n) in sorted(tot.items()): out.append({"source": sid, "group": grp, "mae": sa / n if n else None, "n_players": n}) return out def _lineup_records(graded, sids, by_id, W, where="lineups", ai=False) -> list[dict]: """Own #1 lineup record rows (with ai=True, the AI Division's rows, which add valid and stale rates). An AI lineup that isn't legal scores 0% of perfect and doesn't cash: DraftKings wouldn't have accepted it. A tool's illegal lineup is a capture problem and isn't graded. """ def invalid_ai(value_if_graded): def fill(m, g): if m.get("kind") == "ai" and m.get("valid") is False and value_if_graded(g): return 0.0 return None return fill has_perfect = invalid_ai(lambda g: bool(_get(g, "optimal.points"))) has_cash_line = invalid_ai(lambda g: g.get("cash_line") is not None) base_specs = [ ("pct_perfect", ("mean", "pct_perfect"), has_perfect), ("cash_rate", ("mean", "cashed"), has_cash_line), ("field_percentile", ("mean", "field_percentile"), None), ("top1_rate", ("mean", "top1"), None), ("dupes", ("mean", "dupes"), None), ("donuts", ("mean", "donuts"), None), ("overlap", ("mean", "overlap"), None), ] out = [] for sid in sids: kind = by_id[sid]["kind"] specs = base_specs if ai: specs = [("valid_rate", ("mean", "valid"), None), ("stale_rate", ("mean", "stale"), None)] + base_specs metrics = [] for name, spec, fill in specs: X, D = _matrix(graded, [sid], spec, where=where, fill=fill) st = _pool(W, X, D)[0] if st["n_slates"]: metrics.append({"metric": name, **st}) # ROI: total winnings over total entry fees, when payouts were recorded. Reported, never headlined. fees = wins = 0.0 n_fee = 0 for g in graded: m = g.get(where, {}).get(sid) if m and m.get("entry_fee"): fees += m["entry_fee"] wins += m.get("winnings", 0.0) n_fee += 1 if fees: metrics.append({"metric": "roi", "value": wins / fees - 1, "lo": None, "hi": None, "n_slates": n_fee}) if metrics: out.append({"source": sid, "name": by_id[sid]["name"], "kind": kind, "metrics": metrics}) return out def _form(graded, sids, by_id, window: int) -> list[dict]: """Beat-the-Baseline per slate over the last `window` slates, the pooled last-window number with its range, and the change since the first window (for Most Improved).""" out = [] for k, sid in enumerate(sids): mine = [g for g in graded if g["sources"].get(sid, {}).get("btb") is not None] if not mine: continue series = [{"slate": g["slate"], "date": g["date"], "btb": g["sources"][sid]["btb"]} for g in mine] def window_stats(gs, seed): X = np.array([[g["sources"][sid]["sum_abs"]] for g in gs]) D = np.array([[g["sources"][sid]["sum_abs_base"]] for g in gs]) Wl = _weights(len(gs), seed) st = _pool(Wl, X, D, one_minus=True)[0] boot = 1 - (Wl @ X)[:, 0] / (Wl @ D)[:, 0] return st, boot recent, rboot = window_stats(mine[-window:], [SEED, k, 1]) entry = {"source": sid, "name": by_id[sid]["name"], "kind": by_id[sid]["kind"], "last": series[-window:], "recent": recent, "first": None, "change": None} if len(mine) >= 2 * window: first, fboot = window_stats(mine[:window], [SEED, k, 2]) diff = rboot - fboot entry["first"] = first entry["change"] = {"value": recent["value"] - first["value"], "lo": float(np.percentile(diff, 2.5)), "hi": float(np.percentile(diff, 97.5))} out.append(entry) return out def _awards(bench: dict, cost: list[dict], form: list[dict], cfg: dict, any_ranked: bool) -> list[dict]: """Where each season award stands today. Awards are only final at season end.""" out = [] for a in AWARDS: res = {"id": a["id"], "name": a["name"], "rule": a.get("rule"), "holder": None, "tied": []} if a["id"] == "best_value_for_money": b = bench["btb"] top = [b["leader"]] if b["status"] == "leader" else b["tied"] priced = [c for c in cost if c["source"] in top and c["graded_per_slate"] is not None] if b["status"] in ("leader", "tied") and priced: priced.sort(key=lambda c: c["graded_per_slate"]) if len(priced) == 1 or priced[0]["graded_per_slate"] < priced[1]["graded_per_slate"]: res["holder"] = priced[0]["source"] else: res["tied"] = [c["source"] for c in priced if c["graded_per_slate"] == priced[0]["graded_per_slate"]] elif a["id"] == "most_improved": rows = [f for f in form if f["change"] and f["kind"] in RANKED_KINDS] if any_ranked and len(rows) >= 2: rows.sort(key=lambda f: -f["change"]["value"]) if rows[0]["change"]["lo"] > rows[1]["change"]["hi"]: res["holder"] = rows[0]["source"] else: res["tied"] = [f["source"] for f in rows if f["change"]["hi"] >= rows[0]["change"]["lo"]] else: b = bench[a["benchmark"]] if b["status"] == "leader": res["holder"] = b["leader"] elif b["status"] == "tied": res["tied"] = b["tied"] out.append(res) return out # --------------------------------------------------------------------------- reported, never ranked def _cell(W: np.ndarray, X: np.ndarray, D: np.ndarray) -> dict: st = _pool(W, X, D)[0] if st["value"] is None: return None st["clear"] = bool(st["lo"] is not None and (st["lo"] > 0 or st["hi"] < 0)) # its range leaves out zero return st def _lean(graded: list[dict], sids: list[str], by_id: dict, W: np.ndarray, cfg: dict) -> dict: """Tendencies: average signed error (projected minus actual) overall, by position group and by salary tier, and ownership's (all players and the most-owned), each with its 95% range. Positive = projected too high.""" groups = list(dict.fromkeys(cfg["position_groups"].values())) tiers = [lab for _, _, lab in cfg.get("salary_tiers") or []] S = len(graded) def cell(sid, *keys): # keys, not a dotted path: tier labels like "$5K–$7.9K" contain dots X, D = np.zeros((S, 1)), np.zeros((S, 1)) for i, g in enumerate(graded): v = g.get("sources", {}).get(sid, {}).get("lean") or {} for k in keys: v = v.get(k) if isinstance(v, dict) else None if v and v[0]: D[i, 0], X[i, 0] = float(v[0]), float(v[1]) return _cell(W, X, D) if D.sum() else None rows = [] for sid in sids: if not any((g.get("sources", {}).get(sid) or {}).get("lean") for g in graded): continue s = by_id[sid] rows.append({ "source": sid, "name": s["name"], "kind": s["kind"], "all": cell(sid, "all"), "groups": {grp: cell(sid, "groups", grp) for grp in groups}, "tiers": {t: cell(sid, "tiers", t) for t in tiers}, "own": cell(sid, "own"), "own_chalk": cell(sid, "own_chalk"), }) return {"unit": "pts", "own_unit": "own_pts", "groups": groups, "tiers": tiers, "rows": rows, "chalk_definition": f"the {cfg['chalk_top'] * 2} most-owned players in each slate's graded pool"} def _overlap(graded: list[dict], sids: list[str], by_id: dict, W: np.ndarray) -> dict: """Shared mistakes: how alike two sources' own mistakes are, once the surprises every source shares are taken out. A source's error is its own mistake minus the surprise (what actually happened against what could be known). The surprise is the same for every source, so the covariance of two sources' errors is the surprise's variance plus their shared mistakes. Plain error correlation would mostly measure the surprise, and make the most accurate tools look alike. The surprise's variance is estimated as the median ranked pair's covariance, so 0 means no more alike than the typical pair of tools and 1 means the same mistakes. Needs three ranked sources. Independence = 1 minus a source's average (positive) shared mistakes with the other ranked sources. """ S = len(graded) present = [sid for sid in sids if any(sid in g.get("sources", {}) for g in graded)] ranked = [sid for sid in present if by_id[sid]["kind"] in RANKED_KINDS] refs = [sid for sid in present if by_id[sid]["kind"] not in RANKED_KINDS] empty = {"sources": ranked, "references": refs, "pairs": [], "independence": [], "method": "median-pair"} if len(ranked) < 3 or S == 0: return empty def sums(a, b): M = np.zeros((S, 6)) for i, g in enumerate(graded): ov = g.get("overlap") or {} v = ov.get(f"{a}|{b}") if v is None and ov.get(f"{b}|{a}") is not None: u = ov[f"{b}|{a}"] v = [u[0], u[2], u[1], u[4], u[3], u[5]] # stored the other way round if v and v[0]: M[i] = v return M pairs_all = [(a, b) for a, b in combinations(ranked, 2)] + [(a, r) for a in ranked for r in refs] point, boot = {}, {} for a, b in pairs_all: M = sums(a, b) if not M[:, 0].any(): continue point[(a, b)] = (M.sum(0), int((M[:, 0] > 0).sum())) boot[(a, b)] = (W @ M) if len(W) else np.full((1, 6), np.nan) def moments(t): # t: (..., 6) sums -> covariance and the two variances n, sx, sy, sxx, syy, sxy = (t[..., k] for k in range(6)) with np.errstate(invalid="ignore", divide="ignore"): return (sxy - sx * sy / n) / n, (sxx - sx ** 2 / n) / n, (syy - sy ** 2 / n) / n def shared(stats): """stats: {pair: sums}, each (6,) or (N_BOOT, 6). Returns {pair: rho} with the same leading shape.""" rp = [k for k in stats if k[0] in ranked and k[1] in ranked] cov = {k: moments(stats[k]) for k in stats} s2 = np.median(np.stack([cov[k][0] for k in rp]), axis=0) out = {} with np.errstate(invalid="ignore", divide="ignore"): for k, (c, va, vb) in cov.items(): den = np.sqrt(np.clip(va - s2, 0, None) * np.clip(vb - s2, 0, None)) out[k] = np.where(den > 0, (c - s2) / np.where(den > 0, den, 1), np.nan) return out rho = shared({k: v[0] for k, v in point.items()}) rho_b = shared(boot) pairs = [] for k in point: bt = rho_b[k][np.isfinite(rho_b[k])] lo, hi = (np.percentile(bt, [2.5, 97.5]) if len(bt) else (np.nan, np.nan)) pairs.append({"a": k[0], "b": k[1], "value": float(rho[k]), "lo": float(lo), "hi": float(hi), "n_slates": point[k][1], "reference": k[1] in refs}) independence = [] for sid in ranked: mine = [k for k in point if sid in k and k[0] in ranked and k[1] in ranked] val = 1 - float(np.mean([max(0.0, float(rho[k])) for k in mine])) bt = 1 - np.nanmean(np.stack([np.clip(rho_b[k], 0, None) for k in mine]), axis=0) bt = bt[np.isfinite(bt)] lo, hi = (np.percentile(bt, [2.5, 97.5]) if len(bt) else (np.nan, np.nan)) independence.append({"source": sid, "name": by_id[sid]["name"], "kind": by_id[sid]["kind"], "value": val, "lo": float(lo), "hi": float(hi)}) independence.sort(key=lambda r: -r["value"]) return {**empty, "pairs": pairs, "independence": independence} def _misses(graded: list[dict], sids: list[str], by_id: dict, cfg: dict, top: int = 10) -> dict: """Industry misses: players every graded source missed by the floor or more. Published as counts, bands (10+, 15+, ...) and directions; never as any source's projection.""" allm = [(g, m) for g in graded for m in (g.get("industry_misses") or [])] count = {sid: 0 for sid in sids} for _, m in allm: for sid in m.get("closest", []): if sid in count: count[sid] += 1 present = [sid for sid in sids if any(sid in g.get("sources", {}) for g in graded)] closest = [{"source": sid, "name": by_id[sid]["name"], "kind": by_id[sid]["kind"], "count": count[sid], "share": count[sid] / len(allm) if allm else None} for sid in present] closest.sort(key=lambda r: -r["count"]) allm.sort(key=lambda gm: -gm[1]["min_miss"]) return { "floor": cfg.get("miss_floor", 12), "total": len(allm), "per_slate": len(allm) / len(graded) if graded else None, "under": sum(m["direction"] == "under" for _, m in allm), "over": sum(m["direction"] == "over" for _, m in allm), "split": sum(m["direction"] == "split" for _, m in allm), "closest": closest, "top": [miss_public(m, g) for g, m in allm[:top]], } def miss_band(x: float) -> str: return f"{int(x // 5 * 5)}+" def miss_public(m: dict, g: dict | None = None) -> dict: out = {k: m[k] for k in ("name", "team", "pos", "salary", "actual", "direction")} out["band"] = miss_band(m["min_miss"]) if g: out["date"], out["slate"] = g["date"], g["slate"] return out