From ed0ed3f4b27ae6f9e8b27f9b9f645f35ed17acec Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Mon, 12 Oct 2026 06:17:27 -0700 Subject: [PATCH] research: buy-the-dip studies, both STOP (kept as evidence) dip_buying: SPY/QQQ RSI(2) dip is real after publication (SPY 2008-26 46 bp/trade, t 3.4) but in the market 14% of days, so 3x earns the same Sharpe as SPY; trend-filtered stock RSI(2) breaks even at 10 bp/side and lowers the three-sleeve book (1.14 -> 0.95); stocks far below their high rebound only in market-wide crashes. dip_leverage_overlay: +1x SPY inside the dip windows on the overnight_book books leaves Sharpe flat or lower (0.98 -> 0.99, 1.38 -> 1.33); the gain ranks 77-88% against trend-matched random windows. The books already earn the bounce at about 2x SPY's beta. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 + research/lab/dip_buying.py | 493 +++++++++++++++++++++++++++ research/lab/dip_leverage_overlay.py | 341 ++++++++++++++++++ 3 files changed, 836 insertions(+) create mode 100644 research/lab/dip_buying.py create mode 100644 research/lab/dip_leverage_overlay.py diff --git a/CLAUDE.md b/CLAUDE.md index 193f1d5..d74f5cd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,6 +106,8 @@ PYTHONPATH=. uv run python -m research.lab.overnight_book # three-slee PYTHONPATH=. uv run python -m research.lab.overnight_book_mf [--no-proxy] # the overnight books + a DBMF/CTA sleeve, levered to 50%/yr (preregistered; KEEP 2026-10-09: levered MaxDD unchanged, no CTA sleeve) PYTHONPATH=. uv run python -m research.lab.daytime_reversal # buy the S&P 500's biggest overnight losers at 09:35, sell at the close (preregistered; STOP 2026-10-09: gross 1-6 bp/day, net Sharpe ~0) PYTHONPATH=. uv run python -m research.lab.overnight_midsmall # the overnight-only momentum book on S&P 400/600 (preregistered; STOP 2026-10-09: 0.93 net at 3 bp/side, corr 0.83 with the S&P 500 book) +PYTHONPATH=. uv run --with numba python -m research.lab.dip_buying # buy-the-dip: SPY/QQQ RSI(2), trend-filtered stock RSI(2), drawdown tables (preregistered; STOP 2026-10-12: SPY 46 bp/trade t 3.4 but 14% of days, 3x Sharpe 0.57 = SPY; stocks breakeven 10 bp/side) +PYTHONPATH=. uv run python -m research.lab.dip_leverage_overlay # +1x SPY only inside SPY RSI(2) dip windows, on the overnight_book books (preregistered; STOP 2026-10-12: Sharpe 0.98->0.99 / 1.38->1.33, placebo rank 77-88%; the books already earn the bounce at ~2x SPY beta) uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs) uv run python -m research.lab.fetch_us_fundamentals --universe small # the same for S&P 600 (IJR) PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] [--universe small] # S&P 400 sleeves vs IJH; S&P 600 replication vs IJR (all REJECT 2026-10-08) diff --git a/research/lab/dip_buying.py b/research/lab/dip_buying.py new file mode 100644 index 0000000..73894da --- /dev/null +++ b/research/lab/dip_buying.py @@ -0,0 +1,493 @@ +""" +Quick check: does buying dips pay? The repo's reversal tests so far were +cross-sectional (rev_5/rev_21/residual/sector reversal top 20, the hedged +5-day S&P 400/600 reversal, the 09:35 gap-down buy). This one tests the two +dip-buying rules the project has not tried, plus descriptive tables of what +follows a drawdown. + + PYTHONPATH=. uv run --with numba python -m research.lab.dip_buying + +A. Index dip (Connors & Alvarez 2008, "RSI(2)"), SPY (gated) and QQQ (reported): + enter close > 200-day mean and RSI(2) < 10 (Wilder smoothing) + exit the first close above the 5-day mean + fills next_open (gated): act at the open after the signal close; + close (reported): act at the signal close, i.e. an MOC order sent + with a 15:40 estimate of the signal + otherwise T-bills (FRED DTB3) + costs 1 bp a side (SPY/QQQ/UPRO spreads); 3 bp stress reported + 3x in-trade return 3 r - 2 rf - 0.9%/yr (a UPRO-like fund) + pre-publication check (reported): ^GSPC 1962-1993, close fills, no costs + +B. Stock dip with a trend filter, PIT S&P 500, $5+: + enter close > 200-day mean and RSI(2) < 5; the lowest RSI(2) first, + 10 slots of 10% of equity + exit the first close above the 5-day mean, 10 sessions held, or the + name leaves the index; next_open fills (gated) / close (reported) + costs 10 bp a side (the lab convention); breakeven reported + also the repo's MeanReversionStrategy rule on the lab engine (12-month + return > 0, the 20 biggest 1-month losers, monthly), reported only + 2006-2013 of this panel is survivor-biased (CLAUDE.md), which flatters a + dip buyer most of all: the gate uses 2014-2026. + +C. Descriptive, no gate: forward returns by drawdown from the 252-day high, + for ^GSPC (1928-2026) and ^NDX (1986-2026), and for PIT S&P 500 stocks + (excess over the member mean, entry at the next open). + +The rule is written to prereg.json before the run; the script refuses to +overwrite results.json. + +Result (2026-10-12): STOP. C1 and C2 pass; C3, C4 and C5 fail. + - Index RSI(2) is real but small. It is invested 14% of days, about 8 trades a year. + SPY next_open trade bp t win 1x cagr/xSh 3x cagr/xSh/MaxDD SPY cagr/xSh + 1994-2007 73 4.7 78% 9.6% / 0.81 20.3% / 0.82 / -23% 10.4% / 0.43 + 2008-2026 46 3.4 72% 4.9% / 0.56 10.9% / 0.57 / -41% 11.3% / 0.57 + 2017-2026 39 2.4 74% 5.6% / 0.50 10.9% / 0.51 / -33% 15.3% / 0.74 + Close fills: 2008-26 53 bp (t 4.1), 3x 13.2% / 0.67. QQQ is similar + (46 bp, t 3.4; 3x 0.49 vs QQQ 0.72). ^GSPC 1962-1993: 3.7 bp, t 0.36, + so the effect is absent before the 1990s. + - Reported only: always 1x, switched to 3x inside the dip trades. SPY + 2008-26: 17.5% / 0.69 / -52%, vs 11.3% / 0.57 / -52% for SPY. 2017-26: + 21.0% / 0.78 vs 15.3% / 0.74. QQQ 2008-26: 22.5% / 0.75 vs 16.3% / 0.72. + - Stock RSI(2), 2014-26, next_open: -0.2%/yr, xSh -0.07. There were 5730 + trades with a 60% win rate and +0.8 bp net each; breakeven is 10.4 bp a + side. With close fills: 2.1% / 0.09, breakeven 13 bp. 2006-13 with close + fills (survivor-biased): 15.7% / 0.58. Adding it as a fourth sleeve + lowers the book from 1.14 to 0.95. MeanReversionStrategy (monthly): + 2014-26 7.8% / 0.38, 2006-13 -1.5%, corr 0.89 with SPY. + - Drawdown tables: + - ^GSPC 1988-2026, 252-day forward return: 10.3% unconditional, 4.4% + from 10-20% off the high, 10.8% from 20-30% off, and 27.9% from more + than 30% off. The last bucket is 11 episodes from about 3 bear markets. + - 1928-2026, more than 30% off: 3.6% mean, p5 -61%. + - NDX more than 30% off: 7.7% vs 17.4% unconditional. + - Stocks: results.json averages each bucket over days, and every bucket + comes out negative, which is an aggregation artifact. Pooled stock-days + (stock_dd_pooled.json; stock_dd_table now reports pooled_excess): names more than 30% off their high beat the + member mean by 0.7% over 63 days. That is +3.0% in 2008-09 and 2020 + and -0.5% on all other days. +Interpretation: a short-term index dip is followed by a bounce that is still +measurable after publication. Because the rule is in the market so rarely, +it is worth something only as a leverage timer, not as a sleeve. A stock +dip earns ~20 bp gross per 4-day trade, about what trading it costs. A +stock that is deep in its own drawdown rebounds only when the whole market +crashed with it. Otherwise it keeps lagging, which is momentum. +""" + +from __future__ import annotations + +import json +import os +import pickle +import warnings + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS, load +from research.lab.levered_book import stats, tbill +from research.lab.us_final import monthly_blend + +warnings.filterwarnings("ignore") + +OUT = os.path.join(RESULTS, "dip_buying") +END = "2026-09-24" +FEE3 = 0.009 +IDX_COST, IDX_STRESS, STK_COST = 0.0001, 0.0003, 0.001 +IDX_PERIODS = {"1994-2007": ("1994-01-01", "2007-12-31"), "2008-2026": ("2008-01-01", END), + "2017-2026": ("2017-01-01", END)} +STK_SPAN = ("2006-10-02", END) +STK_GATE = ("2014-01-02", END) +BUCKETS = [(0.0, -0.05), (-0.05, -0.10), (-0.10, -0.20), (-0.20, -0.30), (-0.30, -1.01)] + +PREREG = { + "question": "Does buying short-term dips (index RSI(2) or trend-filtered stock RSI(2)) earn a " + "return worth trading or adding to the three-sleeve book?", + "continue_if": { + "index (all of)": [ + "C1 SPY rsi2 next_open 2008-2026: mean net trade return > 0 with t >= 2.0", + "C2 SPY rsi2 next_open 2017-2026: mean net trade return > 0 with t >= 1.5", + "C3 SPY rsi2 next_open 3x 2008-2026: excess Sharpe >= 0.6 and above SPY buy-and-hold", + ], + "stock (all of)": [ + "C4 stock rsi2 next_open 2014-2026: excess Sharpe >= 0.8 and above the EW S&P 500 member book", + "C5 2014-2026: momentum/lowrisk/trend + stock rsi2 at 1/4 each raises the three-sleeve " + "book's excess Sharpe by >= 0.05", + ], + }, + "decision": { + "CONTINUE (index)": "preregister an index-dip leverage overlay; signals from 15:40 Alpaca bars", + "CONTINUE (stock)": "register the stock dip sleeve in the web engine and replay it", + "STOP": "drop dip buying; the descriptive tables stay as the answer", + }, +} + + +def rsi(c, n: int = 2): + d = c.diff() + up = d.clip(lower=0).ewm(alpha=1 / n, adjust=False).mean() + dn = (-d).clip(lower=0).ewm(alpha=1 / n, adjust=False).mean() + return 100 - 100 / (1 + up / dn) + + +def tstat(x) -> float: + x = np.asarray(x, float) + return float(x.mean() / x.std(ddof=1) * np.sqrt(len(x))) if len(x) > 2 else float("nan") + + +# --------------------------------------------------------------------------- +# A. index dip +# --------------------------------------------------------------------------- + +def index_rule(c: pd.Series, o: pd.Series, th: float = 10.0, fill: str = "next_open"): + """Per-day in-market return (NaN when flat), trade flags and the trade list.""" + enter = ((c > c.rolling(200).mean()) & (rsi(c) < th)).to_numpy() + leave = (c > c.rolling(5).mean()).to_numpy() + cv, ov = c.to_numpy(), o.to_numpy() + n = len(c) + m = np.full(n, np.nan) + traded = np.zeros(n) + trades, held, pend, t0, legs = [], False, None, None, [] + for i in range(1, n): + if fill == "next_open": + if pend == "buy": + held, t0, legs = True, i, [cv[i] / ov[i] - 1] + m[i], traded[i] = legs[0], 1 + elif pend == "sell": + m[i], traded[i] = ov[i] / cv[i - 1] - 1, 1 + legs.append(m[i]) + trades.append((c.index[t0], c.index[i], float(np.prod(1 + np.array(legs)) - 1), i - t0)) + held = False + elif held: + m[i] = cv[i] / cv[i - 1] - 1 + legs.append(m[i]) + pend = None + if held and leave[i]: + pend = "sell" + elif not held and enter[i]: + pend = "buy" + else: + if held: + m[i] = cv[i] / cv[i - 1] - 1 + legs.append(m[i]) + if leave[i]: + traded[i] = 1 + trades.append((c.index[t0], c.index[i], float(np.prod(1 + np.array(legs)) - 1), i - t0)) + held = False + continue + if not held and enter[i]: + held, t0, legs, traded[i] = True, i, [], 1 + return pd.Series(m, c.index), pd.Series(traded, c.index), trades + + +def wrap(m: pd.Series, traded: pd.Series, rf: pd.Series, lev: float, cost: float) -> pd.Series: + rfd = rf / 252 + fee = FEE3 / 252 if lev > 1 else 0.0 + inside = lev * m - (lev - 1) * rfd - fee + return inside.where(m.notna(), rfd) - cost * traded + + +def trade_stats(trades, a, b, cost) -> dict: + r = np.array([x[2] - 2 * cost for x in trades if pd.Timestamp(a) <= x[0] <= pd.Timestamp(b)]) + days = [x[3] for x in trades if pd.Timestamp(a) <= x[0] <= pd.Timestamp(b)] + if len(r) < 3: + return {"n": int(len(r))} + return {"n": int(len(r)), "win": float((r > 0).mean()), "mean_bp": float(r.mean() * 1e4), + "t": tstat(r), "avg_days": float(np.mean(days)), "per_year": float(len(r) / ((pd.Timestamp(b) - pd.Timestamp(a)).days / 365.25))} + + +def brief(r: pd.Series, rf: pd.Series) -> dict: + s = stats(r, rf) + return {k: float(s[k]) for k in ("cagr", "sharpe", "vol", "mdd", "worst_yr")} + + +def part_a(L: dict) -> dict: + out = {} + for tk in ("SPY", "QQQ"): + c = L["close"][tk].dropna().loc[:END] + o = L["open"][tk].reindex(c.index) + rf = tbill(c.index).fillna(0.0) + r_bh = c.pct_change().fillna(0.0) + bh3 = 3 * r_bh - 2 * rf / 252 - FEE3 / 252 + for fill in ("next_open", "close"): + m, tr, trades = index_rule(c, o, fill=fill) + key = f"{tk}_rsi2_{fill}" + rows = {} + for per, (a, b) in IDX_PERIODS.items(): + if c.index[0] > pd.Timestamp(a) - pd.Timedelta(days=300): + continue + sl = slice(a, b) + rows[per] = { + "trades": trade_stats(trades, a, b, IDX_COST), + "time_in": float(m.loc[sl].notna().mean()), + "x1": brief(wrap(m, tr, rf, 1, IDX_COST).loc[sl], rf), + "x3": brief(wrap(m, tr, rf, 3, IDX_COST).loc[sl], rf), + "x3_stress": brief(wrap(m, tr, rf, 3, IDX_STRESS).loc[sl], rf), + "bh": brief(r_bh.loc[sl], rf), "bh3": brief(bh3.loc[sl], rf), + # always 1x, 3x inside dips (switch SPY -> UPRO): reported only + "bh_plus_dip3": brief((r_bh + (2 * (m - rf / 252) - FEE3 / 252).where(m.notna(), 0.0) + - IDX_COST * tr).loc[sl], rf), + } + out[key] = rows + if tk == "SPY" and fill == "next_open": + out["_spy_dip_x1"] = wrap(m, tr, rf, 1, IDX_COST) + out["_spy_dip_x3"] = wrap(m, tr, rf, 3, IDX_COST) + g = L["close"]["^GSPC"].dropna().loc[:"1993-12-31"] + m, tr, trades = index_rule(g, g, fill="close") + rf = tbill(g.index).fillna(0.0) + out["GSPC_rsi2_close_1962-1993"] = {"trades": trade_stats(trades, "1962-01-01", "1993-12-31", 0.0), + "time_in": float(m.loc["1962":].notna().mean())} + return out + + +# --------------------------------------------------------------------------- +# B. stock dip +# --------------------------------------------------------------------------- + +def stock_rule(p, fill: str = "next_open", slots: int = 10, th: float = 5.0, max_hold: int = 10, + cost: float = STK_COST): + cf = p.close.ffill(limit=5) + enter = p.member & p.close.notna() & (p.close >= 5) & (cf > cf.rolling(200, min_periods=200).mean()) \ + & (rsi(cf) < th) + score = rsi(cf).where(enter).to_numpy() + leave = (cf > cf.rolling(5).mean()).to_numpy() + C, O, M = p.close.ffill().to_numpy(), p.open.to_numpy(), p.member.to_numpy() + raw_c = p.close.to_numpy() + cash_r = p.cash_ret.reindex(p.dates).fillna(0.0).to_numpy() + n = len(p.dates) + cash, pos = 1.0, {} # j -> [shares, entry index, entry value] + eq = np.ones(n) + pend_sell, pend_buy, trades = set(), [], [] + stale = {} + + def sell(j, px, i): + nonlocal cash + sh, t0, v0 = pos.pop(j) + cash += sh * px * (1 - cost) + trades.append((p.dates[t0], i - t0, sh * px * (1 - cost) / v0 - 1)) + + def buy(j, px, budget, i): + nonlocal cash + amt = min(budget, cash) + if amt < 0.5 * budget or not np.isfinite(px) or px <= 0: + return + cash -= amt + pos[j] = [amt / (px * (1 + cost)), i, amt] + + for i in range(1, n): + cash *= 1 + cash_r[i] + if fill == "next_open": + for j in list(pend_sell): + if np.isfinite(O[i, j]): + sell(j, O[i, j], i) + pend_sell.discard(j) + budget = eq[i - 1] / slots + for j in pend_buy: + if j not in pos and len(pos) < slots: + buy(j, O[i, j], budget, i) + pend_buy = [] + for j in list(pos): # settle names with no print for 5 sessions + stale[j] = 0 if np.isfinite(raw_c[i, j]) else stale.get(j, 0) + 1 + if stale[j] >= 5: + sell(j, C[i, j], i) + pend_sell.discard(j) + val = cash + sum(sh * C[i, j] for j, (sh, _, _) in pos.items()) + exits = [j for j, (_, t0, _) in pos.items() + if leave[i, j] or i - t0 >= max_hold or not M[i, j]] + if fill == "close": + for j in exits: + if np.isfinite(raw_c[i, j]): + sell(j, raw_c[i, j], i) + else: + pend_sell.update(exits) + free = slots - (len(pos) - len(pend_sell & set(pos))) + row = score[i] + cand = np.where(np.isfinite(row))[0] + cand = [j for j in cand[np.argsort(row[cand])] if j not in pos][:max(free, 0)] + if fill == "close": + for j in cand: + buy(j, raw_c[i, j], val / slots, i) + else: + pend_buy = cand + eq[i] = cash + sum(sh * C[i, j] for j, (sh, _, _) in pos.items()) + eqs = pd.Series(eq, p.dates) + return eqs.pct_change().fillna(0.0), trades + + +def part_b(p, sleeves: pd.DataFrame) -> dict: + from research.lab.us_sweep import run + sp = p.slice("2005-06-01", END) + rf = tbill(sp.dates).fillna(0.0) + out, R = {}, {} + for fill in ("next_open", "close"): + r, trades = stock_rule(sp, fill=fill) + R[f"stock_rsi2_{fill}"] = r.loc[STK_SPAN[0]:] + rows = {} + for per, (a, b) in {"2006-2013": (STK_SPAN[0], "2013-12-31"), "2014-2026": STK_GATE}.items(): + ts = [x for x in trades if pd.Timestamp(a) <= x[0] <= pd.Timestamp(b)] + net = np.array([x[2] for x in ts]) + gross = net + 2 * STK_COST + rows[per] = {**brief(r.loc[a:b], rf), "n_trades": len(ts), "win": float((net > 0).mean()), + "mean_net_bp": float(net.mean() * 1e4), "t": tstat(net), + "breakeven_bp_side": float(gross.mean() / 2 * 1e4), + "avg_days": float(np.mean([x[1] for x in ts]))} + out[f"stock_rsi2_{fill}"] = rows + c = sp.close.ffill(limit=5) + mr = (-(c / c.shift(21) - 1)).where((c / c.shift(252) - 1 > 0) & sp.member) + st = run(sp, mr, [(STK_SPAN[0], "2013-12-31"), STK_GATE], top_n=20, buffer=0.0, every=21) + R["mean_reversion_monthly"] = st["_rets"] + out["mean_reversion_monthly"] = {"2006-2013": brief(st["_rets"].loc[:"2013-12-31"], rf), + "2014-2026": brief(st["_rets"].loc[STK_GATE[0]:], rf), + "turn": float(st["turn"])} + R = pd.DataFrame(R) + S = sleeves.join(R, how="left") + gate = S.loc[STK_GATE[0]:STK_GATE[1]] + rfg = tbill(gate.index).fillna(0.0) + out["ew_sp500_2014-2026"] = brief(gate["ew_sp500"], rfg) + out["spy_2014-2026"] = brief(gate["spy"], rfg) + base = gate[["momentum", "lowrisk", "trend"]].fillna(0.0) + three = monthly_blend(base, {"momentum": 1 / 3, "lowrisk": 1 / 3, "trend": 1 / 3}) + out["book3_2014-2026"] = brief(three, rfg) + mon = lambda x: (1 + x).resample("ME").prod() - 1 + for k in R.columns: + four = monthly_blend(gate[["momentum", "lowrisk", "trend", k]].fillna(0.0), + {"momentum": .25, "lowrisk": .25, "trend": .25, k: .25}) + out[f"book4_{k}_2014-2026"] = brief(four, rfg) + out[f"corr_{k}"] = {s: float(mon(gate[k]).corr(mon(gate[s]))) for s in ("momentum", "lowrisk", "trend", "spy")} + out["_R"] = R + return out + + +# --------------------------------------------------------------------------- +# C. descriptive drawdown tables +# --------------------------------------------------------------------------- + +def bucket_of(dd: pd.DataFrame | pd.Series): + lab = dd * np.nan + for k, (hi, lo) in enumerate(BUCKETS): + lab = lab.mask((dd <= hi) & (dd > lo), k) + return lab + + +def index_dd_table(c: pd.Series, a: str, b: str) -> list[dict]: + c = c.dropna() + dd = c / c.rolling(252, min_periods=200).max() - 1 + lab = bucket_of(dd).loc[a:b] + rows = [] + for k, (hi, lo) in enumerate(BUCKETS + [(None, None)]): + sel = lab.notna() if hi is None else lab == k + row = {"bucket": "all" if hi is None else f"{-hi:.0%}..{-lo if lo > -1 else 1:.0%}", + "share_days": float(sel.mean()), + "episodes": int((sel & ~sel.shift(1, fill_value=False)).sum())} + for h in (21, 63, 252): + f = (c.shift(-h) / c - 1).loc[a:b][sel].dropna() + row[f"fwd{h}"] = {"mean": float(f.mean()), "median": float(f.median()), "p_up": float((f > 0).mean()), + "p5": float(f.quantile(0.05))} if len(f) else {} + rows.append(row) + return rows + + +def stock_dd_table(p, a: str, b: str) -> list[dict]: + c = p.close.ffill(limit=5) + dd = c / c.rolling(252, min_periods=200).max() - 1 + lab = bucket_of(dd).where(p.member & (p.close >= 5)) + later = p.close.ffill() # a delisted name keeps its last print + entry = p.open.shift(-1) + rows = [] + for h in (21, 63): + fwd = later.shift(-h) / entry - 1 + fwd = fwd.where(lab.notna()) + ex = fwd.sub(fwd.mean(axis=1), axis=0) + for k, (hi, lo) in enumerate(BUCKETS): + daily = ex.where(lab == k).mean(axis=1).loc[a:b].dropna() + spaced = daily.iloc[::h] # non-overlapping for the t-stat + rows.append({"h": h, "bucket": f"{-hi:.0%}..{-lo if lo > -1 else 1:.0%}", + "share": float((lab == k).loc[a:b].sum().sum() / lab.loc[a:b].notna().sum().sum()), + "excess_mean": float(daily.mean()), "t_nonoverlap": tstat(spaced), + "pooled_excess": float(ex.where(lab == k).loc[a:b].stack().mean()), + "p_beats_mean": float((ex.where(lab == k).loc[a:b].stack() > 0).mean())}) + return rows + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + with open(os.path.join("data", "research", "long_history.pkl"), "rb") as fh: + L = pickle.load(fh) + res = {"index": part_a(L)} + spy_dip = pd.DataFrame({"spy_dip_x1": res["index"].pop("_spy_dip_x1"), + "spy_dip_x3": res["index"].pop("_spy_dip_x3")}) + for k, rows in res["index"].items(): + if k.startswith("_"): + continue + if "trades" in rows: + print(f"{k}: {rows['trades']} time in {rows['time_in']:.1%}") + continue + for per, r in rows.items(): + t = r["trades"] + print(f"{k:22s} {per}: n {t['n']:3d} ({t.get('per_year', 0):.1f}/yr) win {t.get('win', 0):.0%} " + f"{t.get('mean_bp', 0):6.1f} bp/trade t {t.get('t', 0):4.2f} days {t.get('avg_days', 0):.1f} " + f"in {r['time_in']:.0%}") + for s in ("x1", "x3", "x3_stress", "bh", "bh3", "bh_plus_dip3"): + x = r[s] + print(f" {s:13s} cagr {x['cagr']*100:5.1f}% xsh {x['sharpe']:5.2f} vol {x['vol']*100:4.1f}% " + f"mdd {x['mdd']*100:6.1f}% worst yr {x['worst_yr']*100:6.1f}%") + + res["index_dd"] = {"GSPC_1928-2026": index_dd_table(L["close"]["^GSPC"], "1928-12-31", END), + "GSPC_1988-2026": index_dd_table(L["close"]["^GSPC"], "1988-01-01", END), + "NDX_1986-2026": index_dd_table(L["close"]["^NDX"], "1986-10-01", END)} + for k, rows in res["index_dd"].items(): + print(f"\n{k}: forward return by drawdown from the 252-day high") + for r in rows: + f = " ".join(f"{h}d {r[f'fwd{h}']['mean']*100:6.1f}%/{r[f'fwd{h}']['median']*100:5.1f}% " + f"up {r[f'fwd{h}']['p_up']:.0%} p5 {r[f'fwd{h}']['p5']*100:6.1f}%" + for h in (21, 63, 252) if r[f"fwd{h}"]) + print(f" {r['bucket']:>9s} days {r['share_days']:5.1%} ep {r['episodes']:4d} | {f}") + + p, _ = load("us") + sleeves = pd.read_pickle(os.path.join(RESULTS, "us_final_sleeves.pkl")) + b = part_b(p, sleeves) + R = b.pop("_R") + res["stock"] = b + for k, v in b.items(): + print(f"{k}: {json.dumps(v, default=float)}") + res["stock_dd"] = {"2006-2026": stock_dd_table(p, STK_SPAN[0], END), "2014-2026": stock_dd_table(p, *STK_GATE)} + for k, rows in res["stock_dd"].items(): + print(f"\nstocks {k}: excess return over the member mean by drawdown from the 252-day high") + for r in rows: + print(f" h{r['h']:2d} {r['bucket']:>9s} share {r['share']:5.1%} excess {r['excess_mean']*100:6.2f}% " + f"t {r['t_nonoverlap']:5.2f} beats mean {r['p_beats_mean']:.0%}") + + spy = res["index"]["SPY_rsi2_next_open"] + gate = b["stock_rsi2_next_open"]["2014-2026"] + checks = { + "C1": spy["2008-2026"]["trades"]["mean_bp"] > 0 and spy["2008-2026"]["trades"]["t"] >= 2.0, + "C2": spy["2017-2026"]["trades"]["mean_bp"] > 0 and spy["2017-2026"]["trades"]["t"] >= 1.5, + "C3": spy["2008-2026"]["x3"]["sharpe"] >= 0.6 and spy["2008-2026"]["x3"]["sharpe"] > spy["2008-2026"]["bh"]["sharpe"], + "C4": gate["sharpe"] >= 0.8 and gate["sharpe"] > b["ew_sp500_2014-2026"]["sharpe"], + "C5": b["book4_stock_rsi2_next_open_2014-2026"]["sharpe"] >= b["book3_2014-2026"]["sharpe"] + 0.05, + } + idx_ok = checks["C1"] and checks["C2"] and checks["C3"] + stk_ok = checks["C4"] and checks["C5"] + decision = "CONTINUE (index)" if idx_ok else "" + decision += (" + " if decision and stk_ok else "") + ("CONTINUE (stock)" if stk_ok else "") + res["verdict"] = {"checks": checks, "decision": decision or "STOP"} + print(f"\nchecks {checks} -> {res['verdict']['decision']}") + pd.concat([R, spy_dip], axis=1).to_pickle(os.path.join(OUT, "returns.pkl")) + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2, default=float) + from research.lab.trials import append_trials + rows = [{"name": f"dip_buying:{k}:{per}:{s}", **v[per][s]} for k, v in res["index"].items() + if "trades" not in v for per in v for s in ("x1", "x3")] + rows += [{"name": f"dip_buying:{k}:{per}", **{kk: vv for kk, vv in v[per].items()}} + for k, v in b.items() if k.startswith("stock_rsi2") for per in v] + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), rows, + {"study": "dip_buying", "role": "preregistered quick check"}) + + +if __name__ == "__main__": + main() diff --git a/research/lab/dip_leverage_overlay.py b/research/lab/dip_leverage_overlay.py new file mode 100644 index 0000000..02f5f9f --- /dev/null +++ b/research/lab/dip_leverage_overlay.py @@ -0,0 +1,341 @@ +""" +Does adding index exposure during short-term SPY dips improve the US books? +research/lab/dip_buying.py found that the SPY RSI(2) dip is real (46 bp a +trade, t 3.4, 2008-26) but is in the market on only 14% of days. Its +reported-only row "SPY 1x, 3x inside dips" raised SPY's Sharpe from 0.57 to +0.69. This study puts that overlay on top of the books instead of SPY. + + PYTHONPATH=. uv run python -m research.lab.dip_leverage_overlay + +Books (daily returns, costs inside; research/lab/overnight_book.py books.pkl): + three_sleeve_v7 momentum + lowrisk + trend_v7 (the live us_three_sleeve) gated + three_sleeve_overnight overnight-only momentum + lowrisk + trend_v7 (paper test) gated + two_sleeve_overnight_trend reported + +Signal: dip_buying.index_rule on SPY. Enter when the close is above its +200-day mean and RSI(2) < 10. Exit at the first close above the 5-day mean. + +Overlay (gated: k = 1, next_open fills): book + k x SPY from the open after +the signal close to the open after the exit close, as an MES / margin +position. It is financed at T-bill + 1% for every day touched, including +both partial days, and pays 1 bp a side on k. + reported k = 0.5 and 2; close fills; financing at T-bill + 3%; QQQ + signal and instrument; "book scale", which levers the whole + book (1 + k) inside the windows, with 10 bp a side and either + close timing (optimistic) or only full days inside next_open + windows (conservative) + baselines - the same average SPY exposure held all the time (no timing) + - P1: the same windows shifted circularly by a random offset + - P2 (gated): the same windows at random non-overlapping + starts whose signal day also has SPY above its 200-day + mean. P2 separates the dip timing from the trend filter. + 500 draws each, seed 20261012. + +Span 2006-10-02 .. 2026-09-24, halves split at 2017 (overnight_book.HALVES). +All of it was inspected before, and the SPY version of this overlay was seen +on 2008-26 before this rule was written. This is a consistency check on the +books, not out-of-sample evidence. The rule is written to prereg.json before +the run; the script refuses to overwrite results.json. + +Result (2026-10-12): STOP. No gated book passes. three_sleeve_v7 fails D1 +and D3; three_sleeve_overnight fails D1, D2 and D3. + + The overlay alone (k = 1): 160 windows, 14% of days, +3.2%/yr, vol 6.4%, + self-financed Sharpe 0.50, positive in 15 of 21 years (2011 and 2018 -6.7%). + + book Sharpe + SPY k1 (halves) MaxDD P2 rank constant SPY + three_sleeve_v7 0.98 0.99 (0.75 / 1.28) -37.2 -> -37.2% 88% 0.94 + three_sleeve_overnight 1.38 1.33 (1.14 / 1.55) -16.5 -> -20.4% 77% 1.31 + two_sleeve_overnight_trend 1.39 1.38 (1.13 / 1.64) -20.7 -> -20.7% 84% 1.36 + + - k = 1 adds about 3.3pp of CAGR to each book, but vol rises by the same + proportion, so the Sharpe ratio does not move. k = 2 lowers it (0.94 / + 1.20 / 1.29). k = 0.5 is flat. Close fills give +0.04 / -0.01 / +0.02. + Financing at +3% costs 0.02. A QQQ signal and instrument is worse. + - Dip timing beats random trend-up timing. The P2 placebo gives -0.05 to + -0.10 Sharpe; the real gain is -0.05 to +0.01. That ranks 77-88%, under + the 95% bar. It also beats holding the same average SPY exposure all the + time (+0.02 to +0.05 Sharpe, smaller MaxDD) because the 200-day filter + keeps it out of bear markets. + - Why it adds nothing: the books already earn the bounce. Inside the + windows the books earn 50-64%/yr, against 13-22% outside. Their + correlation with the SPY leg there is 0.57-0.75, and they move about + 2x SPY, through TrendRider's 3x ETFs and high-beta momentum names. A + Sharpe-0.5 leg correlated with a Sharpe-1.4 book lowers the book's + Sharpe. + - Book scale (reported) helps only the weakest book, and only with close + timing: v7 0.98 -> 1.04. With full days only it is 0.95. Overnight: + 1.36 / 1.29. + - Leverage for 50%/yr: + three_sleeve_overnight 2.35x, MaxDD -36.7%, worst yr -18.6% + with the overlay 2.02x, -38.4%, -9.3% + two_sleeve_overnight_trend 1.73x, -34.8%, -24.3% + with the overlay 1.54x, -31.6%, -16.8% + three_sleeve_v7 about 2.9x, -79% (grid check after the + run), against 3.6x, -86% without it + The overlay swaps one kind of leverage for another, at about the same + risk. +""" + +from __future__ import annotations + +import json +import os +import pickle + +import numpy as np +import pandas as pd + +from research.lab.dip_buying import index_rule +from research.lab.factor_scan import RESULTS +from research.lab.levered_book import stats, tbill +from research.lab.overnight_book import HALVES, SPAN, leverage_for, levered + +OUT = os.path.join(RESULTS, "dip_leverage_overlay") +BOOKS_PKL = os.path.join(RESULTS, "overnight_book", "books.pkl") +GATED = ["three_sleeve_v7", "three_sleeve_overnight"] +REPORTED = ["two_sleeve_overnight_trend"] +K = 1.0 +SPREAD, SPREAD_STRESS = 0.01, 0.03 +IDX_COST, BOOK_COST = 0.0001, 0.001 +N_PLACEBO, SEED = 500, 20261012 + +PREREG = { + "question": "Does +k x SPY held only inside SPY RSI(2) dip windows raise a book's Sharpe, beyond what " + "the same exposure at random trend-up times gives?", + "span": SPAN, "halves": HALVES, "gated_books": GATED, "reported_books": REPORTED, + "overlay": "k = 1, next_open fills, financing T-bill + 1% on every day touched, 1 bp a side", + "gates_per_book": [ + "D1 excess Sharpe (book + overlay) >= book + 0.05 over the span, and higher than the book in each half", + "D2 MaxDD (book + overlay) >= book MaxDD - 3pp", + "D3 the span Sharpe gain > 95th percentile of P2 (trend-matched random windows, 500 draws)", + ], + "decision": { + "CONTINUE": "a gated book passes D1-D3 -> paper-trade the overlay alongside that book from a " + "15:40 signal (Alpaca 1-min SPY bars), then decide on live use", + "STOP": "no gated book passes -> do not add a dip overlay; leverage stays static or vol-managed", + }, + "reported": "k 0.5/2, close fills, financing stress, QQQ, book scale, constant-exposure baseline, P1, " + "leverage for 50%/yr with and without the overlay, yearly overlay P&L, book return inside " + "vs outside the windows", + "data_status": "2006-2026 previously inspected; the SPY overlay result was seen before this rule", +} + + +def windows(c: pd.Series, o: pd.Series, fill: str) -> list[tuple[int, int]]: + """(a, b) positions: next_open holds from the open of a to the open of b; + close holds from the close of a to the close of b.""" + _, _, trades = index_rule(c, o, fill=fill) + pos = {d: i for i, d in enumerate(c.index)} + return [(pos[t[0]], pos[t[1]]) for t in trades] + + +def overlay_parts(cv: np.ndarray, ov: np.ndarray, wins, fill: str): + """In-market index return, financed-day flag and trade count per day.""" + n = len(cv) + rcc = np.r_[0.0, cv[1:] / cv[:-1] - 1] + m, inside, tr = np.zeros(n), np.zeros(n), np.zeros(n) + for a, b in wins: + if fill == "next_open": + m[a] += cv[a] / ov[a] - 1 + m[a + 1:b] += rcc[a + 1:b] + m[b] += ov[b] / cv[b - 1] - 1 + inside[a:b + 1] = 1 + else: + m[a + 1:b + 1] += rcc[a + 1:b + 1] + inside[a + 1:b + 1] = 1 + tr[a] += 1 + tr[b] += 1 + return m, inside, tr + + +def overlay(parts, rfd: np.ndarray, k: float, spread: float = SPREAD, cost: float = IDX_COST) -> np.ndarray: + m, inside, tr = parts + return k * (m - inside * (rfd + spread / 252)) - k * cost * tr + + +def xsh(r: np.ndarray, rfd: np.ndarray) -> float: + return float((r - rfd).mean() / r.std() * np.sqrt(252)) + + +def summary(r: pd.Series, rf: pd.Series) -> dict: + s = stats(r, rf) + d = {k: float(s[k]) for k in ("cagr", "sharpe", "vol", "mdd", "worst_yr")} + d["halves"] = [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES] + return d + + +def placebo_trend(rng, wins, elig, lo, hi): + """Same window lengths at random non-overlapping starts with a trend-up signal day.""" + cand = np.where(elig[lo:hi - 1])[0] + lo + taken = np.zeros(hi + 2, bool) + out = [] + for D in rng.permutation([b - a for a, b in wins]): + for _ in range(200): + a = int(rng.choice(cand)) + 1 + b = a + D + if b >= hi or taken[max(a - 1, 0):b + 2].any(): + continue + taken[a:b + 1] = True + out.append((a, b)) + break + return out + + +def placebo_shift(rng, wins, lo, hi): + s = int(rng.integers(63, hi - lo - 63)) + out = [] + for a, b in wins: + a2 = lo + (a - lo + s) % (hi - lo) + b2 = a2 + (b - a) + if b2 < hi: + out.append((a2, b2)) + return out + + +def fmt(name: str, d: dict) -> str: + return (f"{name:46s} cagr {d['cagr']*100:5.1f}% xsh {d['sharpe']:5.2f} vol {d['vol']*100:4.1f}% " + f"mdd {d['mdd']*100:6.1f}% worst yr {d['worst_yr']*100:6.1f}% halves " + f"{d['halves'][0]:5.2f} {d['halves'][1]:5.2f}") + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + B = pd.read_pickle(BOOKS_PKL).loc[SPAN[0]:SPAN[1]] + idx = B.index + with open(os.path.join("data", "research", "long_history.pkl"), "rb") as fh: + L = pickle.load(fh) + rf = tbill(idx).fillna(0.0) + rfd = (rf / 252).to_numpy() + + inst = {} + for tk in ("SPY", "QQQ"): + c = L["close"][tk].dropna().loc[:SPAN[1]] + o = L["open"][tk].reindex(c.index) + miss = idx.difference(c.index) + assert len(miss) == 0, f"{tk} lacks {len(miss)} book dates" + sma = c.rolling(200).mean() + # windows on the full history, then positions in the book's calendar + on_idx = lambda w: [(idx.get_loc(c.index[a]), idx.get_loc(c.index[b])) for a, b in w + if c.index[a] >= idx[0] and c.index[b] in idx] + cv, ov = c.reindex(idx).to_numpy(), o.reindex(idx).to_numpy() + inst[tk] = {"cv": cv, "ov": ov, "elig": (c > sma).reindex(idx).fillna(False).to_numpy(), + "w": {f: on_idx(windows(c, o, f)) for f in ("next_open", "close")}} + spy = inst["SPY"] + parts = {f: overlay_parts(spy["cv"], spy["ov"], spy["w"][f], f) for f in ("next_open", "close")} + ov1 = overlay(parts["next_open"], rfd, K) + share_in = float(parts["next_open"][1].mean()) + + res = {"n_windows": len(spy["w"]["next_open"]), "share_days": share_in, "overlay": {}, "books": {}} + ovs = pd.Series(ov1, idx) + res["overlay"] = {"ann_mean": float(ovs.mean() * 252), "vol": float(ovs.std() * np.sqrt(252)), + "sharpe_self_financed": float(ovs.mean() / ovs.std() * np.sqrt(252)), + "yearly": {int(y): float(v) for y, v in ovs.groupby(idx.year).sum().items()}} + yrs = pd.Series(res["overlay"]["yearly"]) + print(f"windows {res['n_windows']}, share of days {share_in:.1%}; overlay k=1 mean {res['overlay']['ann_mean']*100:.2f}%/yr " + f"vol {res['overlay']['vol']*100:.1f}% self-financed Sharpe {res['overlay']['sharpe_self_financed']:.2f}; " + f"positive years {(yrs > 0).sum()}/{len(yrs)}") + print("overlay P&L by year %:", (yrs * 100).round(1).to_dict()) + + rng = np.random.default_rng(SEED) + lo, hi = 0, len(idx) + P2 = [overlay(overlay_parts(spy["cv"], spy["ov"], placebo_trend(rng, spy["w"]["next_open"], spy["elig"], lo, hi), + "next_open"), rfd, K) for _ in range(N_PLACEBO)] + P1 = [overlay(overlay_parts(spy["cv"], spy["ov"], placebo_shift(rng, spy["w"]["next_open"], lo, hi), + "next_open"), rfd, K) for _ in range(N_PLACEBO)] + m_in = parts["next_open"][1] > 0 + r_spy_cc = np.r_[0.0, spy["cv"][1:] / spy["cv"][:-1] - 1] + const = share_in * K * (r_spy_cc - (rfd + SPREAD / 252)) + + gates = {} + for name in GATED + REPORTED: + r = B[name].to_numpy() + base = summary(B[name], rf) + row = {"book": base, "variants": {}} + print(f"\n=== {name}") + print(fmt("book", base)) + variants = { + "spy k1 next_open": r + ov1, + "spy k0.5 next_open": r + overlay(parts["next_open"], rfd, 0.5), + "spy k2 next_open": r + overlay(parts["next_open"], rfd, 2.0), + "spy k1 close": r + overlay(parts["close"], rfd, K), + "spy k1 next_open fin+3%": r + overlay(parts["next_open"], rfd, K, SPREAD_STRESS), + "constant SPY, same avg exposure": r + const, + } + q = inst["QQQ"] + variants["qqq k1 next_open"] = r + overlay(overlay_parts(q["cv"], q["ov"], q["w"]["next_open"], "next_open"), + rfd, K) + # book scale: lever the book itself inside the windows + _, in_close, tr_close = parts["close"] + variants["book scale k1 close timing"] = (r * (1 + K * in_close) - K * in_close * (rfd + SPREAD / 252) + - K * BOOK_COST * tr_close) + full_days = np.zeros(len(r)) + for a, b in spy["w"]["next_open"]: + full_days[a + 1:b] = 1 + variants["book scale k1 full days only"] = (r * (1 + K * full_days) - K * parts["next_open"][1] * (rfd + SPREAD / 252) + - K * BOOK_COST * parts["next_open"][2]) + for vname, x in variants.items(): + d = summary(pd.Series(x, idx), rf) + row["variants"][vname] = d + print(fmt(" " + vname, d)) + s0 = xsh(r, rfd) + g2 = np.array([xsh(r + p, rfd) - s0 for p in P2]) + g1 = np.array([xsh(r + p, rfd) - s0 for p in P1]) + gain = row["variants"]["spy k1 next_open"]["sharpe"] - base["sharpe"] + row["placebo"] = {"gain": gain, "P2_mean": float(g2.mean()), "P2_p95": float(np.quantile(g2, 0.95)), + "P2_pct_rank": float((g2 < gain).mean()), "P1_mean": float(g1.mean()), + "P1_p95": float(np.quantile(g1, 0.95)), "P1_pct_rank": float((g1 < gain).mean())} + print(f" Sharpe gain {gain:+.3f} | P2 trend-matched mean {g2.mean():+.3f} p95 {np.quantile(g2, .95):+.3f} " + f"rank {row['placebo']['P2_pct_rank']:.1%} | P1 shift mean {g1.mean():+.3f} p95 {np.quantile(g1, .95):+.3f} " + f"rank {row['placebo']['P1_pct_rank']:.1%}") + ins, outs = r[m_in], r[~m_in] + row["book_inside_vs_outside"] = {"inside_ann": float(ins.mean() * 252), "outside_ann": float(outs.mean() * 252), + "inside_vol": float(ins.std() * np.sqrt(252)), + "outside_vol": float(outs.std() * np.sqrt(252)), + "corr_with_overlay_inside": float(np.corrcoef(ins, ov1[m_in])[0, 1])} + bi = row["book_inside_vs_outside"] + print(f" book inside windows {bi['inside_ann']*100:5.1f}%/yr (vol {bi['inside_vol']*100:4.1f}%) vs outside " + f"{bi['outside_ann']*100:5.1f}%/yr (vol {bi['outside_vol']*100:4.1f}%); corr with SPY leg inside " + f"{bi['corr_with_overlay_inside']:.2f}") + lev = {} + for lname, x in (("book", B[name]), ("book + spy k1", pd.Series(r + ov1, idx))): + Lx = leverage_for(x, rf) + d = {"L": Lx} + if np.isfinite(Lx): + s = stats(levered(x, rf, Lx), rf) + d.update({"mdd": float(s["mdd"]), "worst_yr": float(s["worst_yr"]), "vol": float(s["vol"]), + "halves_cagr": [float(stats(levered(x.loc[a:b], rf, Lx), rf)["cagr"]) for a, b in HALVES]}) + print(f" 50%/yr: {lname:14s} L {Lx:4.2f} mdd {d['mdd']*100:6.1f}% worst yr {d['worst_yr']*100:6.1f}% " + f"halves cagr {d['halves_cagr'][0]*100:5.1f}% {d['halves_cagr'][1]*100:5.1f}%") + lev[lname] = d + row["leverage_for_50"] = lev + res["books"][name] = row + if name in GATED: + v = row["variants"]["spy k1 next_open"] + gates[name] = { + "D1": v["sharpe"] >= base["sharpe"] + 0.05 and all(v["halves"][i] > base["halves"][i] for i in range(2)), + "D2": v["mdd"] >= base["mdd"] - 0.03, + "D3": gain > row["placebo"]["P2_p95"], + } + passing = [k for k, g in gates.items() if all(g.values())] + res["verdict"] = {"gates": gates, "passing": passing, "decision": "CONTINUE" if passing else "STOP"} + print(f"\ngates {gates}\n-> {res['verdict']['decision']} {passing}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.DataFrame({"overlay_spy_k1": ov1, "inside": parts["next_open"][1]}, index=idx).to_pickle( + os.path.join(OUT, "overlay.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"dip_leverage_overlay:{b}:{v}", **{x: y for x, y in d.items() if x != "halves"}} + for b, row in res["books"].items() for v, d in row["variants"].items()], + {"study": "dip_leverage_overlay", "role": "preregistered consistency check"}) + + +if __name__ == "__main__": + main()