""" Does overnight_recovery_top10 gain from ranking S&P 400 names alongside the S&P 500? LITE ranked #1-4 on this score from 2025-07 while it was an S&P 400 member, and the live book could only buy it after it joined the S&P 500 on 2026-03-23 (at ~8x the 2025-07 price). PYTHONPATH=. uv run python -m research.lab.universe_midcap Both arms run on one merged panel (S&P 500 panel + S&P 400 panel), with one engine, one cost (20 bp a side, the mid-cap panel's cost) and the live construction (overnight momentum + distance from the 126-day low, top 10, every 10 days, buffer 2, next-open fills, phase-averaged over the 10 offsets). Only the eligible set differs: base PIT S&P 500 members (ranks among them) union PIT S&P 500 or S&P 400 members (ranks among the union) union_adv20 union, but names under $20m 60-day median dollar volume are not eligible union_minus_top3 union without the three S&P-400-only names that added most S&P 400 membership comes from iShares IJH month-end holdings (up to a month late). Yahoo lacks most delisted mid caps: coverage is 46-78% of members in 2006-2016 and >= 85% from 2019, so the primary window starts 2019-01-02 and 2007-2018 is shown for reference only (survivor-biased toward the union). The decision rule is written to PREREG before anything runs. Result (2026-10-08): FAIL (3 of 5 checks). Primary window 2019-01..2026-09: arm CAGR Sharpe MaxDD Sharpe 2019-24 base 29.4% 0.98 -34.3% 0.70 union 24.7% 0.86 -38.8% 0.48 union_adv20 23.8% 0.84 -38.6% 0.46 union_minus_top3 17.8% 0.71 -38.6% 0.34 (alpha vs base -6.7%/yr, NW t -2.1) S&P-400-only names fill 31-66% of the union book. Their gain comes from a few winners (PENN 2020, LITE 2025-26, SMCI 2023); the losses are spread over many names (RIG, RUN, GTLS, STRL, ...). Union minus base by year: 2019 -14pp, 2020 +16pp, 2022 -26pp, 2025 +5pp, 2026 YTD +3.5pp. Both known biases favour the union (missing delisted mid caps; 20 bp for every arm), so the verdict stands. Reading: momentum among mid caps is right-skewed and loses on average after costs; the LITE miss is real, but the rule that would have caught it costs more than it earns. 2007-2018 (reference only): union 0.65 vs base 0.57, survivor-biased. """ from __future__ import annotations import json import os import warnings import numpy as np import pandas as pd from research.lab import panel as pnl from research.lab.alpha import alpha_stats from research.lab.engine import Costs, Panel, simulate from research.lab.factors import overnight_mom, rec_126 from research.lab.panel_mid import load_us_mid from research.lab.us_sweep import _blend, run warnings.filterwarnings("ignore") OUT = os.path.join("data", "research", "results", "universe_midcap") PRIMARY = ("2019-01-02", "2026-09-24") NO_2025 = ("2019-01-02", "2024-12-31") REFERENCE = ("2007-01-03", "2018-12-31") BLOCKS = [("2019-01-02", "2020-12-31"), ("2021-01-04", "2022-12-30"), ("2023-01-03", "2024-12-31"), ("2025-01-02", "2026-09-24")] KW = dict(top_n=10, buffer=2.0, every=10) SAME_SECURITY_TOL = 0.02 # median |log(S&P close / mid close)| for one security PREREG = { "question": "Does adding PIT S&P 400 members to the overnight_recovery_top10 universe improve it?", "arms": ["base (PIT S&P 500)", "union (PIT S&P 500 or S&P 400)", "union_adv20", "union_minus_top3"], "engine": "research.lab.engine.simulate via us_sweep.run; top 10, every 10, buffer 2, next-open fills, " "10 phases averaged; 20 bp a side for every arm", "primary_window": PRIMARY, "pass_if_all": [ "primary window: Sharpe(union) - Sharpe(base) >= 0.10", "primary window: MaxDD(union) >= MaxDD(base) - 5pp", "2019-01-02..2024-12-31: Sharpe(union) >= Sharpe(base)", "primary window: Sharpe(union_minus_top3) >= Sharpe(base)", "primary window: sign of Sharpe(union_adv20) - Sharpe(base) equals that of union", ], "on_pass": "propose a forward-only validation book; no production change from this study alone", "reference_only": f"{REFERENCE}: S&P 400 price coverage 46-81%, survivor-biased toward the union", "status": "the primary window was partly seen in earlier live replays and research; not independent OOS", } def merged_panel() -> tuple[Panel, pd.DataFrame, pd.DataFrame, dict]: """S&P 500 panel's calendar, union of tickers. A ticker in both panels keeps the S&P 500 panel's bars, filled from the mid panel where those are missing, when the two agree on common days; otherwise the mid column is kept as a separate security ``@mid``.""" sp, mid = pnl.load_us(), load_us_mid("mid") dates = sp.dates mid_f = {k: getattr(mid, k).reindex(dates) for k in ("open", "high", "low", "close", "volume")} mid_member = mid.member.reindex(dates).fillna(False).astype(bool) rename, mismatched = {}, [] for t in mid.close.columns: if t not in sp.close.columns: continue a, b = sp.close[t], mid_f["close"][t] common = a.notna() & b.notna() if common.sum() < 20: continue if np.nanmedian(np.abs(np.log(a[common] / b[common]))) > SAME_SECURITY_TOL: rename[t] = f"{t}@mid" mismatched.append(t) for k in mid_f: mid_f[k] = mid_f[k].rename(columns=rename) mid_member = mid_member.rename(columns=rename) cols = sorted(set(sp.close.columns) | set(mid_f["close"].columns)) f = {} for k in ("open", "high", "low", "close", "volume"): f[k] = getattr(sp, k).reindex(columns=cols).combine_first(mid_f[k].reindex(columns=cols)) sp_member = sp.member.reindex(columns=cols).fillna(False).astype(bool) mid_member = mid_member.reindex(columns=cols).fillna(False).astype(bool) & f["close"].notna() has_px = f["close"].notna() p = Panel("us_sp500_sp400", f["open"], f["high"], f["low"], f["close"], f["volume"], sp_member | mid_member, has_px, has_px, sp.cash_ret, sp.benchmark, Costs(buy=0.002, sell=0.002, fixed_fee=0.0, capital=10_000.0)) info = {"tickers": len(cols), "overlap": int(len(set(mid.close.columns) & set(sp.close.columns))), "overlap_mismatched": mismatched} return p, sp_member, mid_member & ~sp_member, info def score_for(p: Panel, eligible: pd.DataFrame, factors: dict) -> pd.DataFrame: ranks = [f.where(eligible).rank(axis=1, pct=True) for f in factors.values()] return _blend(*ranks).where(eligible) def window_stats(rets: pd.Series, a: str, b: str) -> dict: r = rets.loc[a:b] e = (1 + r).cumprod() yrs = len(r) / 252 return {"cagr": e.iloc[-1] ** (1 / yrs) - 1, "sharpe": r.mean() / r.std() * np.sqrt(252), "mdd": float((e / e.cummax() - 1).min())} def mid_only_contribution(p: Panel, score: pd.DataFrame, mid_only: pd.DataFrame, a: str, b: str) -> pd.Series: """Phase-averaged sum of daily (weight x return) by ticker while the name was held as an S&P-400-only member (weights at each fill, drifting with price until the next fill).""" r = p.close.ffill().pct_change(fill_method=None).fillna(0.0) w0 = p.dates.searchsorted(pd.Timestamp(a)) sp = p.slice(p.dates[max(w0 - 5, 0)], b) total = pd.Series(0.0, index=p.close.columns) for off in range(KW["every"]): res = simulate(sp, score, offset=off, warmup=5, **KW) h = res.holdings.reindex(sp.dates).ffill().fillna(0.0) growth = (1 + r.loc[sp.dates]).cumprod() anchor = growth.loc[res.holdings.index].reindex(sp.dates).ffill() drift = (h * growth / anchor).fillna(0.0) drift = drift.div(drift.sum(axis=1).where(lambda s: s > 0, 1.0), axis=0) contrib = (drift.shift(1) * r.loc[sp.dates]).loc[a:b] flag = mid_only.reindex(sp.dates).loc[a:b].shift(1, fill_value=False) total += contrib.where(flag, 0.0).sum() return (total / KW["every"]).sort_values(ascending=False) def main() -> None: os.makedirs(OUT, exist_ok=True) prereg_path = os.path.join(OUT, "prereg.json") if os.path.exists(os.path.join(OUT, "results.json")): raise SystemExit(f"{OUT}/results.json exists; this study does not overwrite itself") if not os.path.exists(prereg_path): with open(prereg_path, "w") as fh: json.dump(PREREG, fh, indent=2) p, sp_member, mid_only, info = merged_panel() print(f"merged panel {p.close.shape}, {info}", flush=True) fac = {"overnight_mom": overnight_mom(p), "rec_126": rec_126(p)} vol_d = (p.close * p.volume).rolling(60, min_periods=40).median() elig = {"base": sp_member, "union": p.member, "union_adv20": p.member & (vol_d >= 20e6)} scores = {k: score_for(p, v, fac) for k, v in elig.items()} windows = {"primary": PRIMARY, "no_2025": NO_2025, "reference": REFERENCE} res, rets = {}, {} for arm, sc in scores.items(): out = run(p, sc, [(REFERENCE[0], PRIMARY[1])], **KW) rets[arm] = out["_rets"] print(f"ran {arm}", flush=True) contrib = mid_only_contribution(p, scores["union"], mid_only, *PRIMARY) top3 = [t for t in contrib.index[:3]] elig["union_minus_top3"] = p.member.copy() elig["union_minus_top3"].loc[:, top3] = False scores["union_minus_top3"] = score_for(p, elig["union_minus_top3"], fac) rets["union_minus_top3"] = run(p, scores["union_minus_top3"], [(REFERENCE[0], PRIMARY[1])], **KW)["_rets"] # the run() returns are the tranche-averaged book; report those for arm, r in rets.items(): res[arm] = {w: window_stats(r, *ab) for w, ab in windows.items()} res[arm]["blocks"] = [window_stats(r, *ab)["sharpe"] for ab in BLOCKS] res[arm]["years"] = ((1 + r.loc[PRIMARY[0]:PRIMARY[1]]).groupby( r.loc[PRIMARY[0]:PRIMARY[1]].index.year).prod() - 1).round(4).to_dict() a = alpha_stats(r.loc[PRIMARY[0]:PRIMARY[1]], rets["base"].loc[PRIMARY[0]:PRIMARY[1]]) res[arm]["alpha_vs_base"] = {k: a[k] for k in a if isinstance(a[k], (int, float))} # share of the union book in S&P-400-only names (phase 0, at fills) w0 = p.dates.searchsorted(pd.Timestamp(PRIMARY[0])) one = simulate(p.slice(p.dates[w0 - 5], PRIMARY[1]), scores["union"], offset=0, warmup=5, **KW) flag = mid_only.reindex(one.holdings.index) share = (one.holdings.where(flag, 0.0).sum(axis=1) / one.holdings.sum(axis=1)).fillna(0.0) P, B = res["union"]["primary"], res["base"]["primary"] d_sh = P["sharpe"] - B["sharpe"] checks = { "sharpe_gain_ge_0.10": d_sh >= 0.10, "mdd_not_worse_5pp": P["mdd"] >= B["mdd"] - 0.05, "no_2025_sharpe_ge_base": res["union"]["no_2025"]["sharpe"] >= res["base"]["no_2025"]["sharpe"], "minus_top3_sharpe_ge_base": res["union_minus_top3"]["primary"]["sharpe"] >= B["sharpe"], "adv20_same_sign": np.sign(res["union_adv20"]["primary"]["sharpe"] - B["sharpe"]) == np.sign(d_sh), } verdict = "PASS" if all(checks.values()) else "FAIL" summary = {"prereg": PREREG, "panel": info, "results": res, "mid_only_top_contributors": contrib.head(15).round(4).to_dict(), "mid_only_bottom_contributors": contrib.tail(10).round(4).to_dict(), "top3_removed": top3, "mid_only_share_by_year": share.groupby(share.index.year).mean().round(3).to_dict(), "checks": {k: bool(v) for k, v in checks.items()}, "verdict": verdict} with open(os.path.join(OUT, "results.json"), "w") as fh: json.dump(summary, fh, indent=2, default=float) pd.DataFrame(rets).to_pickle(os.path.join(OUT, "returns.pkl")) from research.lab.trials import append_trials append_trials(os.path.join("data", "research", "results", "trials_us.jsonl"), [{"name": f"universe_midcap:{arm}", **{f"{w}_{k}": v for w in windows for k, v in res[arm][w].items()}} for arm in res], {"study": "universe_midcap", "role": "preregistered comparison; window previously examined"}) for arm in res: s = res[arm] print(f"{arm:18s} " + " ".join( f"{w}: cagr {s[w]['cagr']*100:5.1f}% sh {s[w]['sharpe']:.2f} mdd {s[w]['mdd']*100:5.1f}%" for w in windows) + f" blocks {[round(x, 2) for x in s['blocks']]}") print("mid-only top contributors:", summary["mid_only_top_contributors"]) print("mid-only share by year:", summary["mid_only_share_by_year"]) print("checks:", summary["checks"], "→", verdict) if __name__ == "__main__": main()