diff --git a/CLAUDE.md b/CLAUDE.md index 3bb296c..30bb5da 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,6 +82,8 @@ uv run python -m research.lab.fetch_wiki_attention && PYTHONPATH=. uv run --with PYTHONPATH=. uv run --with numba python -m research.lab.concentration # industry caps on the momentum sleeve PYTHONPATH=. uv run python -m research.lab.universe_midcap # overnight_recovery_top10 on S&P 500 ∪ S&P 400 (preregistered; FAIL 2026-10-08) PYTHONPATH=. uv run --with numba python -m research.lab.stream_portfolio # risk-budgeted book of momentum/lowrisk/TrendRider/ETF TSMOM/BTC streams, 2006-26 (preregistered; REJECT 2026-10-08) +PYTHONPATH=. uv run python -m research.lab.hedged_momentum # momentum with its beta hedged by index futures (quick check; STOP 2026-10-08) +PYTHONPATH=. uv run python -m research.lab.futures_trend [--no-proxy] # 24-market futures TSMOM on Yahoo =F + data checks (quick check; Sharpe 0.44, DATA 2026-10-08) uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs) uv run python -m research.lab.fetch_us_fundamentals --universe small # the same for S&P 600 (IJR) PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] [--universe small] # S&P 400 sleeves vs IJH; S&P 600 replication vs IJR (all REJECT 2026-10-08) diff --git a/research/lab/futures_trend.py b/research/lab/futures_trend.py new file mode 100644 index 0000000..96665aa --- /dev/null +++ b/research/lab/futures_trend.py @@ -0,0 +1,186 @@ +""" +Quick check before building a futures pipeline: do free Yahoo continuous +futures (=F) support a backtest, and does a diversified futures trend book +(Moskowitz, Ooi & Pedersen 2012) add a better stream than the 8-ETF +long-only trend of research/lab/stream_portfolio.py (Sharpe 0.39)? + + PYTHONPATH=. uv run python -m research.lab.futures_trend [--no-proxy] + +Data: Yahoo front-month continuous contracts. They are not back-adjusted, so +each roll shows up as a jump between contracts. Data checks first: + tracking daily corr and annual drift of ES/NQ/ZN/GC vs SPY/QQQ/IEF/GLD + minus the T-bill (futures earn the excess return) + jumps days with |return| > 6x the 63-day vol, per market and year +Strategy (primary): 24 markets (4 equity, 4 rates, 6 FX, 3 metals, +4 energy, 3 grains). Position = sign(252-day return) x 40% / (EWMA-60 vol) +/ N, through t-1. Book return = T-bill + sum(w r), less 3 bp per unit of +notional traded. Jump days are kept as Yahoo prints them, and the +roll-cleaned variant (jumps zeroed) is reported next to it. + +The decision rule is in PREREG and is written before the run; the script +refuses to overwrite results.json. + +Result (2026-10-08): DATA. C1 and C2 fail; C3 passes. + - Tracking: ES 0.979 and NQ 0.978 pass. ZN vs IEF (0.914) and GC vs GLD + (0.888, drift +2.2%/yr) miss the bar. The likely cause is settlement + times (ZN 15:00, GC 13:30 ET vs the ETFs' 16:00 close), not bad prints. + - Roll jumps are rare: 0.0-0.7 days a year per market. + - tsmom24: 7.4% / Sharpe 0.44, vol 15%, MaxDD -34%, halves 0.34 / 0.56. + Correlation is 0.22 with TrendRider and 0.05 with momentum. Returns: + +24.5% in 2008, +30.9% in 2022, -24.8% in 2016. + - Roll-cleaned: 0.48 (0.34 / 0.65). Cleaning does not lift 2006-16, so + better data is unlikely to reach 0.5 in both halves. + - By sector: equity 0.44, metals 0.45, energy 0.27, grains 0.15, rates 0.11, + FX -0.01. + - Exploration afterwards (not preregistered, stream_portfolio blend): + - tsmom24 in place of the ETF xtrend: invvol5 1.07 -> 1.21, equal5 1.23 -> 1.30. + - Added to the three sleeves: 0.96-1.02 -> 1.05-1.07, MaxDD -32/-39% -> -20/-28%. + It is a diversifier (crisis years) with a low stand-alone Sharpe, not a core stream. +""" + +from __future__ import annotations + +import argparse +import json +import os +import pickle + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS +from research.lab.fetch_long_history import fetch_yahoo +from research.lab.levered_book import stats, tbill + +OUT = os.path.join(RESULTS, "futures_trend") +STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl") +MARKETS = { + "equity": ["ES=F", "NQ=F", "YM=F", "RTY=F"], + "rates": ["ZT=F", "ZF=F", "ZN=F", "ZB=F"], + "fx": ["6E=F", "6J=F", "6B=F", "6A=F", "6C=F", "6S=F"], + "metals": ["GC=F", "SI=F", "HG=F"], + "energy": ["CL=F", "NG=F", "HO=F", "RB=F"], + "grains": ["ZC=F", "ZW=F", "ZS=F"], +} +ALL = [t for v in MARKETS.values() for t in v] +TRACK = {"ES=F": "SPY", "NQ=F": "QQQ", "ZN=F": "IEF", "GC=F": "GLD"} +HALVES = [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")] +SPAN = (HALVES[0][0], HALVES[1][1]) +COST = 0.0003 + +PREREG = { + "question": "Is free Yahoo =F data good enough, and does 24-market futures TSMOM beat the ETF trend stream?", + "primary": "tsmom24 (as printed by Yahoo)", + "data_ok_if": "daily corr >= 0.95 for ES/NQ/ZN/GC vs their ETF and |annual drift| <= 2%", + "continue_if_all": [ + "C1 data_ok", + "C2 tsmom24 excess Sharpe >= 0.50 in both halves (2006-16, 2017-26)", + "C3 monthly corr of tsmom24 with TrendRider V10 <= 0.40", + ], + "decision": {"CONTINUE": "buy roll-adjusted data (e.g. Norgate / CSI) and build a futures trend sleeve", + "STOP": "drop TODO 3 as a strategy source", + "DATA": "if C1 fails the strategy numbers are not trusted: decide on buying data alone"}, + "halves": HALVES, +} + + +def jumps(r: pd.Series) -> pd.Series: + vol = r.rolling(63, min_periods=20).std().shift(1) + return r.abs() > 6 * vol + + +def tsmom(R: pd.DataFrame, rf: pd.Series, look: int = 252) -> tuple[pd.Series, pd.Series]: + lvl = (1 + R.fillna(0.0)).cumprod().where(R.notna()) + sig = np.sign(lvl / lvl.shift(look) - 1) + vol = R.ewm(span=60, min_periods=60).std() * np.sqrt(252) + n = sig.notna().sum(axis=1).clip(lower=1) + W = (sig * (0.40 / vol)).div(n, axis=0).shift(1).fillna(0.0) + ret = rf.reindex(R.index) / 252 + (W * R.fillna(0.0)).sum(axis=1) - W.diff().abs().sum(axis=1) * COST + return ret, W.abs().sum(axis=1) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--no-proxy", action="store_true") + if not ap.parse_args().no_proxy: + from research.lab.fetch_us import _proxy_env + _proxy_env() + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + close, _ = fetch_yahoo(ALL, start="2000-01-01") + missing = [t for t in ALL if t not in close or close[t].dropna().empty] + print("missing:", missing) + with open(os.path.join("data", "research", "long_history.pkl"), "rb") as fh: + etf = pickle.load(fh)["close"] + days = etf["SPY"].dropna().loc["2000-01-01":].index + px = close.reindex(columns=[t for t in ALL if t not in missing]).reindex(days) + R = px.pct_change(fill_method=None) + rf = tbill(days) + + res = {"data": {}, "jumps_per_year": {}} + print("\n=== tracking vs ETF (futures earn ETF return - T-bill)") + for f, e in TRACK.items(): + if f in missing: + continue + a = R[f] + b = etf[e].reindex(days).pct_change(fill_method=None) - rf / 252 + ok = a.notna() & b.notna() + corr = float(a[ok].corr(b[ok])) + drift = float((a[ok].mean() - b[ok].mean()) * 252) + res["data"][f] = {"etf": e, "corr": corr, "annual_drift": drift, "start": str(a.first_valid_index().date())} + print(f" {f:6s} vs {e:4s} from {a.first_valid_index().date()} corr {corr:.3f} drift {drift*100:5.1f}%/yr") + data_ok = all(v["corr"] >= 0.95 and abs(v["annual_drift"]) <= 0.02 for v in res["data"].values()) + + print("\n=== jump days (|r| > 6x 63d vol) per year, and first date") + J = R.apply(jumps) + for t in px.columns: + n = int(J[t].sum()) + yrs = max(R[t].notna().sum() / 252, 1) + res["jumps_per_year"][t] = n / yrs + print(f" {t:6s} from {px[t].first_valid_index().date()} jumps {n:3d} ({n/yrs:4.1f}/yr)") + + book, gross = tsmom(R, rf) + clean, _ = tsmom(R.mask(J, 0.0), rf) + groups = {g: tsmom(R[[t for t in v if t in R]], rf)[0] for g, v in MARKETS.items()} + S = pd.read_pickle(STREAMS) + out = {"tsmom24": book, "tsmom24_roll_cleaned": clean, **{f"tsmom_{g}": r for g, r in groups.items()}} + res["streams"] = {} + print("\n=== strategies (excess Sharpe; halves 2006-16 / 2017-26)") + m_tr = (1 + S["trend"].dropna()).resample("ME").prod() - 1 + m_mo = (1 + S["momentum"].dropna()).resample("ME").prod() - 1 + for k, r in {**out, "etf_xtrend": S["xtrend"].dropna(), "trend_v10": S["trend"].dropna()}.items(): + r = r.loc[SPAN[0]:SPAN[1]] + s = stats(r, rf) + h = [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES] + m = (1 + r).resample("ME").prod() - 1 + c_tr, c_mo = float(m.corr(m_tr)), float(m.corr(m_mo)) + res["streams"][k] = {"cagr": float(s["cagr"]), "sharpe": float(s["sharpe"]), "vol": float(s["vol"]), + "mdd": float(s["mdd"]), "halves": h, "corr_trend": c_tr, "corr_momentum": c_mo} + print(f" {k:22s} cagr {s['cagr']*100:5.1f}% xsh {s['sharpe']:5.2f} vol {s['vol']*100:4.1f}% " + f"mdd {s['mdd']*100:6.1f}% halves {h[0]:5.2f} {h[1]:5.2f} corr trend {c_tr:5.2f} mom {c_mo:5.2f}") + print(f" avg gross leverage tsmom24 {gross.loc[SPAN[0]:].mean():.2f}") + yr = (1 + book.loc[SPAN[0]:SPAN[1]]).groupby(book.loc[SPAN[0]:SPAN[1]].index.year).prod() - 1 + print("tsmom24 yearly %:", (yr * 100).round(1).to_dict()) + res["yearly_tsmom24"] = {int(k): float(v) for k, v in yr.items()} + + p = res["streams"]["tsmom24"] + checks = {"C1": data_ok, "C2": all(x >= 0.50 for x in p["halves"]), "C3": p["corr_trend"] <= 0.40} + decision = "CONTINUE" if all(checks.values()) else ("DATA" if not data_ok else "STOP") + res["verdict"] = {"checks": checks, "decision": decision} + print(f"\nchecks {checks} -> {decision}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.DataFrame(out).to_pickle(os.path.join(OUT, "returns.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"futures_trend:{k}", **v} for k, v in res["streams"].items()], + {"study": "futures_trend", "role": "preregistered quick check; both halves previously inspected"}) + + +if __name__ == "__main__": + main() diff --git a/research/lab/hedged_momentum.py b/research/lab/hedged_momentum.py new file mode 100644 index 0000000..5af1872 --- /dev/null +++ b/research/lab/hedged_momentum.py @@ -0,0 +1,145 @@ +""" +Quick check before building a futures-hedge pipeline: is the momentum +sleeve's alpha, with its beta hedged by short index futures (MNQ / MES), a +stream worth holding next to TrendRider? + + PYTHONPATH=. uv run --with numba python -m research.lab.stream_portfolio # once: streams.pkl + PYTHONPATH=. uv run python -m research.lab.hedged_momentum + +Inputs: the cached daily streams of research/lab/stream_portfolio.py +(momentum = overnight_recovery_top10 on the PIT S&P 500, TrendRider V10 on +synthetic 3x ETFs). Index returns are QQQ / SPY total return from +long_history.pkl. + +Hedge: the account holds the stocks with its own cash and is short beta_t +units of index futures; a short future earns -(index - T-bill). So + hedged_t = mom_t - beta_{t-1} (idx_t - rf_t) - |d beta| * 2 bp - 0.5%/yr / 252 +beta_{t-1} is the 126-day OLS beta of momentum on the index, from returns +through t-1. The 0.5%/yr covers roll and basis slippage. Variants: QQQ (MNQ, +primary) or SPY (MES), rolling or fixed beta 1. + +Books (monthly reset, research/lab/us_final.monthly_blend): + trend_raw 1/2 TrendRider + 1/2 momentum + trend_hedged 1/2 TrendRider + 1/2 hedged momentum + trend TrendRider alone + +The decision rule is in PREREG and is written before the run; the script +refuses to overwrite results.json. + +Result (2026-10-08): STOP. C1 and C2 both fail. +Excess Sharpe, 2006-10..2026-09, halves 2006-16 / 2017-26: + - momentum unhedged: 20.3% / 0.69 (0.59 / 0.81). Alpha vs QQQ is +5.3%/yr, + t 1.10, beta 1.04. On the Tiingo-overlaid PIT panel this alpha is about half + the +9.4% (t 1.86) the pre-backfill panel gave. + - hedged, QQQ, rolling beta: 1.8% / 0.11 (0.02 / 0.23), MaxDD -66%. + Beta drifts from 0.5 to 1.8 by year, and the 126-day estimate trails it. + - hedged, SPY, beta 1 (best variant): 10.3% / 0.48 (0.49 / 0.46). + - 1/2 TrendRider + 1/2 momentum: 1.01 (0.63 / 1.47). + With hedged momentum instead: 0.85 (0.37 / 1.42). +Interpretation: about 21% of the hedged stream's vol is idiosyncratic risk +from 10 names, against an alpha of a few percent. Hedging removes the beta +that paid for the momentum sleeve. +""" + +from __future__ import annotations + +import json +import os + +import numpy as np +import pandas as pd + +from research.lab.alpha import alpha_stats, bench_returns +from research.lab.factor_scan import RESULTS +from research.lab.levered_book import stats, tbill +from research.lab.us_final import monthly_blend + +OUT = os.path.join(RESULTS, "hedged_momentum") +STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl") +HALVES = [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")] +SPAN = (HALVES[0][0], HALVES[1][1]) +TRADE, FRICTION = 0.0002, 0.005 + +PREREG = { + "question": "Is beta-hedged momentum (short QQQ futures, rolling 126d beta) a stream worth a futures pipeline?", + "primary": "hedged_qqq_roll126", + "continue_if_all": [ + "C1 hedged_qqq_roll126 excess Sharpe >= 0.50 in both halves", + "C2 trend_hedged Sharpe >= trend_raw Sharpe + 0.10 in both halves", + ], + "decision": {"CONTINUE": "build the futures hedge (MNQ) into the sleeve framework and preregister a book", + "STOP": "drop TODO 2; momentum stays an unhedged long sleeve"}, + "halves": HALVES, "costs": {"hedge_trade": TRADE, "hedge_friction_per_year": FRICTION}, +} + + +def rolling_beta(r: pd.Series, b: pd.Series, n: int = 126) -> pd.Series: + cov = r.rolling(n, min_periods=63).cov(b) + return (cov / b.rolling(n, min_periods=63).var()).shift(1) + + +def hedge(r: pd.Series, idx: pd.Series, rf: pd.Series, beta: pd.Series) -> pd.Series: + beta = beta.fillna(1.0) + return r - beta * (idx - rf / 252) - beta.diff().abs().fillna(0.0) * TRADE - beta * FRICTION / 252 + + +def describe(r: pd.Series, rf: pd.Series, q: pd.Series) -> dict: + s = stats(r.loc[SPAN[0]:SPAN[1]], rf) + a = alpha_stats(r.loc[SPAN[0]:SPAN[1]], q, rf) + return {"cagr": float(s["cagr"]), "sharpe": float(s["sharpe"]), "vol": float(s["vol"]), + "mdd": float(s["mdd"]), "worst_yr": float(s["worst_yr"]), + "halves": [float(stats(r.loc[a_:b_], rf)["sharpe"]) for a_, b_ in HALVES], + "beta_qqq": float(a["beta"]), "alpha_qqq": float(a["alpha"]), "alpha_t": float(a["t"])} + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + R = pd.read_pickle(STREAMS).loc[SPAN[0]:SPAN[1]] + rf = tbill(R.index) + q, s = bench_returns(R.index, "QQQ"), bench_returns(R.index, "SPY") + mom = R["momentum"] + S = {"momentum": mom, "trend": R["trend"], "qqq": q} + for name, idx in (("qqq", q), ("spy", s)): + S[f"hedged_{name}_roll126"] = hedge(mom, idx, rf, rolling_beta(mom, idx)) + S[f"hedged_{name}_beta1"] = hedge(mom, idx, rf, pd.Series(1.0, index=mom.index)) + D = pd.DataFrame(S) + for k, h in (("trend_raw", "momentum"), ("trend_hedged", "hedged_qqq_roll126"), + ("trend_hedged_spy", "hedged_spy_roll126")): + D[k] = monthly_blend(D[["trend", h]].fillna(0.0), {"trend": 0.5, h: 0.5}) + + res = {k: describe(D[k], rf, q) for k in D.columns} + beta = rolling_beta(mom, q) + res["beta_path"] = {str(y): float(v) for y, v in beta.groupby(beta.index.year).mean().items()} + for k, d in res.items(): + if k == "beta_path": + continue + print(f"{k:22s} cagr {d['cagr']*100:5.1f}% xsh {d['sharpe']:5.2f} vol {d['vol']*100:4.1f}% " + f"mdd {d['mdd']*100:6.1f}% halves {d['halves'][0]:5.2f} {d['halves'][1]:5.2f} | " + f"vs QQQ beta {d['beta_qqq']:4.2f} alpha {d['alpha_qqq']*100:5.1f}% t {d['alpha_t']:4.2f}") + print("mean rolling beta to QQQ by year:", {k: round(v, 2) for k, v in res["beta_path"].items()}) + m = (1 + D[["hedged_qqq_roll126", "trend", "momentum", "qqq"]]).resample("ME").prod() - 1 + print("monthly corr:\n", m.corr().round(2).to_string()) + yr = (1 + D[["momentum", "hedged_qqq_roll126", "trend_raw", "trend_hedged", "qqq"]]).groupby(D.index.year).prod() - 1 + print((yr * 100).round(1).to_string()) + + h, tr, th = res["hedged_qqq_roll126"], res["trend_raw"], res["trend_hedged"] + checks = {"C1": all(x >= 0.50 for x in h["halves"]), + "C2": all(th["halves"][i] >= tr["halves"][i] + 0.10 for i in range(2))} + res["verdict"] = {"checks": checks, "decision": "CONTINUE" if all(checks.values()) else "STOP"} + print(f"checks {checks} -> {res['verdict']['decision']}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"hedged_momentum:{k}", **v} for k, v in res.items() if k not in ("beta_path", "verdict")], + {"study": "hedged_momentum", "role": "preregistered quick check; both halves previously inspected"}) + + +if __name__ == "__main__": + main()