diff --git a/CLAUDE.md b/CLAUDE.md index 04d7051..22486e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -122,6 +122,7 @@ PYTHONPATH=. uv run python -m research.lab.fetch_options snapshot|status # dail PYTHONPATH=. uv run python -m research.lab.fetch_options backfill # OPRA daily option bars since 2024-02 for every S&P 500 member since then (resumable, ~2-3 h; pauses 16:10-16:55 NY); then builds volume_panel.csv.gz PYTHONPATH=. uv run python -m research.lab.fetch_options volume-panel # date x underlying option volume (all 0-100 DTE strikes and the 5-100 DTE / 0.7-1.3 band), os_volume, pc_volume PYTHONPATH=. uv run python -m research.lab.auction_audit [fetch] # overnight sleeve: Yahoo open/close vs the primary exchange's opening/closing cross prints (Alpaca SIP trades), 800 nights 2016-26 (preregistered; PASS 2026-10-12: +0.20 bp a side incl. SEC fee, the 1 bp backtest cost is conservative) +PYTHONPATH=. uv run python -m research.lab.no_letf_book # books without leveraged ETFs for the $10k IBKR account: QQQ trend held in QQQ (+ Reg T margin), momentum blends, vs QQQ at matched risk (preregistered; NONE BEATS QQQ 2026-10-13: Sharpe gain < 0.10; trend's gain is crashes, MaxDD -28% vs -83% since 1990; x2 margin 17.0% vs 14.9%) PYTHONPATH=. uv run python -m research.lab.option_overlay [fetch] # leverage routes (Reg T margin, box, MES futures, calls) and S&P 500 puts (Cboe PPUT3M/PPUT held to expiry) for $10k-$100k accounts (preregistered; 2026-10-12: no route reaches 50% under Reg T, cap L 1.71 -> 36%/yr; puts B FAIL +1.0pp) PYTHONPATH=. uv run --with numba python -m research.lab.option_volume # Johnson-So O/S and put/call volume on OPRA bars 2024-03..2026-09 (preregistered; STOP 2026-10-12: O/S sign reversed, t -1.6, corr -0.65 with momentum; put/call t 0.4) uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs) diff --git a/research/lab/no_letf_book.py b/research/lab/no_letf_book.py new file mode 100644 index 0000000..da34eb9 --- /dev/null +++ b/research/lab/no_letf_book.py @@ -0,0 +1,262 @@ +""" +Can a book that holds no leveraged ETF beat QQQ? The live account (2026-10-13) +is $10k at IBKR Pro without the Complex or Leveraged ETP permission, trades +whole shares only (IBKR's API refuses fractional orders), and may borrow on +Reg T margin (at most 2x). The QQQ trend sleeve of us_momentum_qqq_trend held +TQQQ; here it holds QQQ itself and borrows for exposure above 1x. + + PYTHONPATH=. uv run python -m research.lab.fetch_long_history # once + PYTHONPATH=. uv run python -m research.lab.no_letf_book + +Constructions (fixed before the run; no parameter is searched): + qqq QQQ total return, buy and hold: the yardstick + qqq_trend_1x E = 1{QQQ > 200-day average} x min(1.0, 30% / sigma) in QQQ, the + rest in SHY; sigma = QQQ's EWMA vol (20-day half-life). The rule of + strategies/qqq_trend.py (QQQVolTrend) with the cap cut to 1x + qqq_trend_m15 the same with QQQVolTrend's own cap 1.5: exposure above 1x is + borrowed at T-bill + 1.5% (IBKR Pro under $100k) + momentum overnight_recovery_top10 held all day (stream_portfolio streams.pkl) + mom_qqq_1x momentum + qqq_trend_1x, equal capital, month-start reset when a + share is more than 25% off (managed_futures_book.band_blend) + mom_qqq_m15 momentum + qqq_trend_m15 (book exposure at most 1.25x) + +Trend fills: decided on closes through t-1, traded at the close of t ("lag" in +research/lab/vol_trend.py, the live MOC timing); the book trades back to target +when the signal flips or E is more than 25% off; 10 bp per unit traded. Every +construction also pays $1 an order on a $10k account (IBKR Pro Fixed, the +commission the paper account's what-if returned): 2 orders a rebalance for the +trend (QQQ and SHY legs) and 142 a year for momentum (the web engine's 1,250 +orders over 2017-12..2026-10). + +Risk-matched reading: each construction is scaled by L* = (QQQ's volatility / +its own) over the design period, capped so that gross exposure stays within +Reg T's 2x; L* above 1 borrows at T-bill + 1.5%, below 1 leaves cash at the +T-bill rate. L* is fixed on the design period and applied unchanged to the +holdout. + +Periods: QQQ-only constructions design 1990-2016, holdout 2017-2026-09 (QQQ +from ^NDX + 0.5%/yr dividends before 1999-03, research/lab/letf_history.py); +momentum constructions design 2006-10..2016, holdout 2017..2026-09. The +momentum signal was chosen on this panel, the panel is survivor-biased before +about 2014, and us_momentum_qqq_trend's TQQQ version was seen on 2026-09-29: +the holdout is not independent out-of-sample evidence. + +"Beats QQQ" needs all three: + G1 excess Sharpe above QQQ's by at least 0.10 in the design period and the holdout + G2 MaxDD over the construction's span shallower than QQQ's over the same span + G3 risk-matched holdout CAGR above QQQ's holdout CAGR + +The rule is written to prereg.json before the run; the script refuses to +overwrite results.json. + +Result (2026-10-13): NONE BEATS QQQ. G1 (Sharpe +0.10 in both periods) fails +for every construction; the QQQ trends pass G2 and G3. + + construction from CAGR Sharpe design / holdout Sharpe MaxDD L* matched holdout CAGR + qqq (yardstick) 1990 14.9% 0.56 0.47 / 0.86 -83% 1.00 21.6% + qqq_trend_1x 1990 12.1% 0.60 0.51 / 0.85 -28% 1.57 22.1% + qqq_trend_m15 1990 13.8% 0.57 0.47 / 0.85 -36% 1.24 24.0% + momentum 2006 18.6% 0.64 0.54 / 0.77 (QQQ .60/.86) -69% (QQQ -53%) 0.62 16.1% + mom_qqq_1x 2006 14.9% 0.69 0.55 / 0.84 -45% 0.99 19.0% + mom_qqq_m15 2006 16.4% 0.70 0.57 / 0.84 -46% 0.91 19.6% + +The momentum stream is phase-averaged and carries Tiingo's delisted names +(20.3% a year before the $1 orders). The single-phase series of beat_qqq.py is +6pp a year higher with corr 0.98, so us_momentum_qqq_trend's 27.9% vs 21.5% +for 2017-26 (2026-09-29) was optimistic. + +Reported after the verdict (exploratory, not gated): + - The trend's gain is crash avoidance, not Sharpe. 2000-02: +0.2% vs QQQ + -73%; 2008: -8% vs -42%; 2022: -23% vs -33%. 14 orders a year. + - Margin frontier, qqq_trend_1x x L, 1990-2026, borrowing at T-bill + 1.5% + (charged even on the SHY leg, so conservative): + x1.5 14.9% (= QQQ), holdout 21.4%, MaxDD -40% + x2.0 17.0% vs 14.9%, holdout 26.1% vs 21.6%, MaxDD -50% vs -83%, + worst year -43% (2022) + - mom_qqq_1x x1.25 matches QQQ's 2006+ MaxDD (-53%) at 17.2% vs 16.6%. + - QQQ held overnight only (gross, Yahoo opens): Sharpe 0.59 / 1.08 / 1.26 / + 0.80 in 2000-09 / 2010-16 / 2017-20 / 2021-26, intraday -0.35 / 0.35 / + 0.53 / 0.40. Two orders a night cost 1.9% (IBKR Tiered) to 5% (Fixed) a + year at $10k. + - IBKR's API refuses fractional orders (error 10243, what-if 2026-10-13). At + $10k the momentum half holds $500 a name; 5 of the 2026-10-12 top 10 cost + more than that. +""" + +from __future__ import annotations + +import json +import os +import pickle + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS +from research.lab.letf_history import SRC, build +from research.lab.levered_book import stats +from research.lab.managed_futures_book import band_blend +from research.lab.overnight_book import STREAMS + +OUT = os.path.join(RESULTS, "no_letf_book") +END = "2026-09-24" +QQQ_SPAN, MOM_SPAN = ("1990-01-02", END), ("2006-10-02", END) +HOLDOUT = "2017-01-01" +MA, TARGET, HALF_LIFE = 200, 0.30, 20 +COST, BAND = 0.001, 0.25 +SPREAD = 0.015 # IBKR Pro margin: benchmark + 1.5% under $100k +ACCOUNT, PER_ORDER = 10_000.0, 1.0 +MOMENTUM_ORDERS_A_YEAR = 142 +REG_T = 2.0 +PREREG = { + "question": "does a book without leveraged ETFs beat QQQ, raw and at QQQ's volatility", + "constructions": ["qqq", "qqq_trend_1x", "qqq_trend_m15", "momentum", "mom_qqq_1x", "mom_qqq_m15"], + "trend_rule": "1{QQQ > MA200} x min(cap, 30% / EWMA vol hl20), lag fills, 25% band, 10 bp a unit", + "costs": "$1 an order on $10k: trend 2 an rebalance, momentum 142 a year", + "financing": "T-bill + 1.5% on exposure above 1x; cash earns SHY (trend) or T-bill (scaling)", + "risk_matched": "L* = design-period vol(QQQ) / vol(construction), gross exposure <= 2x, fixed for the holdout", + "periods": {"qqq_only": [QQQ_SPAN[0], "2016-12-31", HOLDOUT, END], + "momentum": [MOM_SPAN[0], "2016-12-31", HOLDOUT, END]}, + "gates": {"G1": "Sharpe >= QQQ + 0.10 in design and holdout", + "G2": "MaxDD shallower than QQQ over the same span", + "G3": "risk-matched holdout CAGR > QQQ holdout CAGR"}, + "data_status": "momentum signal and the TQQQ version of the blend were examined before; not independent OOS", +} + + +def trend(syn: pd.DataFrame, rf: pd.Series, cap: float) -> tuple[pd.Series, pd.Series, pd.Series]: + """Daily returns, exposure held and orders sent of the QQQ trend held in QQQ (margin above 1x).""" + px = syn["QQQ"] + r = px.pct_change().fillna(0.0) + on = (px > px.rolling(MA).mean()).astype(float) + sigma = np.sqrt((r ** 2).ewm(halflife=HALF_LIFE, min_periods=40).mean() * 252) + E = (on * (TARGET / sigma).clip(upper=cap)).fillna(0.0) + E.iloc[:MA] = 0.0 + tgt = E.shift(2).fillna(0.0).to_numpy() # held over t: decided at t-2's close, traded at t-1's + rq, rs = r.to_numpy(), syn["SHY"].pct_change().fillna(0.0).to_numpy() + fin = ((rf.reindex(px.index).ffill().fillna(0.0) + SPREAD) / 252).to_numpy() + w = 0.0 + out, held, orders = np.zeros(len(tgt)), np.zeros(len(tgt)), np.zeros(len(tgt)) + for t in range(1, len(tgt)): + g = 1 + w * rq[t] + ((1 - w) * rs[t] if w <= 1 else -(w - 1) * fin[t]) + out[t] = g - 1 + w = w * (1 + rq[t]) / g + want = tgt[t + 1] if t + 1 < len(tgt) else w + if (want == 0) != (w == 0) or abs(want - w) > BAND * max(w, 1e-9): + legs = (want < 1 or w < 1) + 1 # the SHY leg trades too while either side holds cash + out[t] -= abs(want - w) * COST * legs + orders[t] = legs + w = want + held[t] = w + idx = px.index + return pd.Series(out, index=idx), pd.Series(held, index=idx), pd.Series(orders, index=idx) + + +def with_commissions(r: pd.Series, orders: pd.Series, scale: float = 1.0) -> pd.Series: + """$1 an order, charged on the day it is sent, per dollar of a sleeve holding ``scale`` of the $10k account.""" + return r - orders.reindex(r.index).fillna(0.0) * PER_ORDER / (ACCOUNT * scale) + + +def scaled(r: pd.Series, rf: pd.Series, L: float) -> pd.Series: + cash = rf.reindex(r.index).ffill().fillna(0.0) / 252 + return L * r + (1 - L) * cash - max(L - 1, 0.0) * SPREAD / 252 + + +def describe(r: pd.Series, rf: pd.Series) -> dict: + s = stats(r, rf) + yr = (1 + r).groupby(r.index.year).prod() - 1 + return {"cagr": float(s["cagr"]), "sharpe": float(s["sharpe"]), "vol": float(s["vol"]), "mdd": float(s["mdd"]), + "worst_yr": float(yr.min()), "years": {int(y): float(v) for y, v in yr.items()}} + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + with open(SRC, "rb") as fh: + src = pickle.load(fh) + syn, _ = build(src) + fred = src["fred"].ffill() + rf = (fred["DTB3"].reindex(syn.index.union(fred.index)).ffill().reindex(syn.index) / 100).fillna(0.0) + qqq = syn["QQQ"].pct_change().fillna(0.0) + + t1, e1, o1 = trend(syn, rf, 1.0) + t15, e15, o15 = trend(syn, rf, 1.5) + R = pd.DataFrame({"qqq": qqq, "qqq_trend_1x": with_commissions(t1, o1), + "qqq_trend_m15": with_commissions(t15, o15)}) + streams = pd.read_pickle(STREAMS) + mom = streams["momentum"].reindex(R.index) + mom_orders = pd.Series(MOMENTUM_ORDERS_A_YEAR / 252, index=R.index) + R["momentum"] = with_commissions(mom, mom_orders) + span = R.loc[MOM_SPAN[0]:MOM_SPAN[1]] + for name, leg, orders in (("mom_qqq_1x", "qqq_trend_1x", o1), ("mom_qqq_m15", "qqq_trend_m15", o15)): + # Each sleeve already pays its own orders at full-account scale; on half the + # capital the same order count costs twice as much per dollar of the sleeve. + X = pd.DataFrame({"momentum": with_commissions(mom, mom_orders, 0.5), + leg: with_commissions(t1 if leg == "qqq_trend_1x" else t15, orders, 0.5)}).loc[span.index] + R.loc[span.index, name], _ = band_blend(X, rf.reindex(span.index)) + max_exposure = {"qqq": 1.0, "qqq_trend_1x": 1.0, "qqq_trend_m15": 1.5, "momentum": 1.0, + "mom_qqq_1x": 1.0, "mom_qqq_m15": 1.25} + + res = {"constructions": {}, "verdict": {}} + rows = [] + for name in PREREG["constructions"]: + start = QQQ_SPAN[0] if name.startswith("qqq") else MOM_SPAN[0] + r = R[name].loc[start:END].dropna() + q = R["qqq"].loc[r.index] + des, hol = r.loc[:"2016-12-31"], r.loc[HOLDOUT:] + qd, qh = q.loc[:"2016-12-31"], q.loc[HOLDOUT:] + L = min(qd.std() / des.std(), REG_T / max_exposure[name]) + rl = scaled(r, rf, L) + d = {"span": [str(r.index[0].date()), END], "full": describe(r, rf), "design": describe(des, rf), + "holdout": describe(hol, rf), "qqq_full": describe(q, rf), "qqq_design": describe(qd, rf), + "qqq_holdout": describe(qh, rf), "L_star": float(L), + "matched_full": describe(rl, rf), "matched_holdout": describe(rl.loc[HOLDOUT:], rf)} + if name.startswith("qqq_trend"): + e = e1 if name == "qqq_trend_1x" else e15 + o = o1 if name == "qqq_trend_1x" else o15 + d["avg_exposure"] = float(e.loc[r.index].mean()) + d["orders_a_year"] = float(o.loc[r.index].sum() / (len(r) / 252)) + d["crashes"] = {k: float((1 + r.loc[a:b]).prod() - 1) for k, (a, b) in + {"2000-02": ("2000-01-01", "2002-12-31"), "2008": ("2008-01-01", "2008-12-31"), + "2022": ("2022-01-01", "2022-12-31")}.items()} + g = {"G1": d["design"]["sharpe"] >= d["qqq_design"]["sharpe"] + 0.10 + and d["holdout"]["sharpe"] >= d["qqq_holdout"]["sharpe"] + 0.10, + "G2": d["full"]["mdd"] > d["qqq_full"]["mdd"], + "G3": d["matched_holdout"]["cagr"] > d["qqq_holdout"]["cagr"]} + d["gates"] = g + res["constructions"][name] = d + rows.append({"construction": name, "span": d["span"][0][:4], "cagr": d["full"]["cagr"], + "sharpe": d["full"]["sharpe"], "mdd": d["full"]["mdd"], + "sh_des": d["design"]["sharpe"], "sh_hold": d["holdout"]["sharpe"], + "cagr_hold": d["holdout"]["cagr"], "L*": L, "matched_cagr_hold": d["matched_holdout"]["cagr"], + "matched_mdd": d["matched_full"]["mdd"], "qqq_sh_des": d["qqq_design"]["sharpe"], + "qqq_sh_hold": d["qqq_holdout"]["sharpe"], "qqq_cagr_hold": d["qqq_holdout"]["cagr"], + "qqq_mdd": d["qqq_full"]["mdd"], **g}) + T = pd.DataFrame(rows) + pd.set_option("display.width", 250) + print(T.to_string(index=False, float_format=lambda x: f"{x:.3f}")) + for name in ("qqq_trend_1x", "qqq_trend_m15"): + d = res["constructions"][name] + print(f"{name}: avg exposure {d['avg_exposure']:.2f}, {d['orders_a_year']:.1f} orders a year, " + f"crashes {({k: round(v, 3) for k, v in d['crashes'].items()})}") + passing = [n for n, d in res["constructions"].items() if n != "qqq" and all(d["gates"].values())] + res["verdict"] = {"passing": passing, + "decision": "BEATS QQQ: " + ", ".join(passing) if passing else "NONE BEATS QQQ"} + print(f"\n-> {res['verdict']['decision']}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + R.to_pickle(os.path.join(OUT, "daily.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"no_letf_book:{n}", **{k: v for k, v in d["full"].items() if k != "years"}} + for n, d in res["constructions"].items()], + {"study": "no_letf_book", "role": "preregistered"}) + + +if __name__ == "__main__": + main()