From 4899bb970225942940d95fc92151104349fa9c1c Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 9 Oct 2026 22:03:18 -0700 Subject: [PATCH] research: overnight-only momentum in the sleeve book (CONTINUE), CTA sleeve (KEEP) - overnight_book: swapping the all-day momentum sleeve for the overnight-only one lifts the three-sleeve book from Sharpe 0.98 to 1.38 (halves 1.16/1.64) and MaxDD from -37% to -17% at 1 bp a side; at 4 bp the gain is gone. - overnight_book_mf: at the leverage for 50%/yr, a DBMF sleeve leaves the levered MaxDD unchanged (~-36%) and needs more leverage; no CTA sleeve. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 + research/lab/overnight_book.py | 195 ++++++++++++++++++++++++++++++ research/lab/overnight_book_mf.py | 156 ++++++++++++++++++++++++ 3 files changed, 353 insertions(+) create mode 100644 research/lab/overnight_book.py create mode 100644 research/lab/overnight_book_mf.py diff --git a/CLAUDE.md b/CLAUDE.md index db4541c..e7373ca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,6 +100,8 @@ PYTHONPATH=. uv run python -m research.lab.orb_stocks_in_play # 5-min open PYTHONPATH=. uv run python -m research.lab.overnight_only # momentum names held close->open only, MOC/MOO costs (preregistered; STOP 2026-10-09: net Sharpe 1.29-1.42 < 1.5, but the sleeve's whole return is overnight) uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs) uv run python -m research.lab.fetch_us_fundamentals --universe small # the same for S&P 600 (IJR) +PYTHONPATH=. uv run python -m research.lab.overnight_book # three-sleeve book with the overnight-only momentum sleeve, leverage for 50%/yr (preregistered; CONTINUE 2026-10-09: Sharpe 0.98 -> 1.38, MaxDD -37% -> -17% at 1 bp/side; 4 bp/side erases it) +PYTHONPATH=. uv run python -m research.lab.overnight_book_mf [--no-proxy] # the overnight books + a DBMF/CTA sleeve, levered to 50%/yr (preregistered; KEEP 2026-10-09: levered MaxDD unchanged, no CTA sleeve) PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] [--universe small] # S&P 400 sleeves vs IJH; S&P 600 replication vs IJR (all REJECT 2026-10-08) PYTHONPATH=. uv run python -m research.lab.fetch_shortvolume # FINRA REG SHO daily short volume (~8y archive, resume-safe) PYTHONPATH=. uv run python -m research.lab.short_volume [us|midsmall|all] # SVR (short-volume ratio) IC / quintile scan on PIT panels diff --git a/research/lab/overnight_book.py b/research/lab/overnight_book.py new file mode 100644 index 0000000..4f6bd7b --- /dev/null +++ b/research/lab/overnight_book.py @@ -0,0 +1,195 @@ +""" +Does the live three-sleeve book get better when its momentum sleeve holds +only overnight (research/lab/overnight_only.py), and how much leverage would +the book need for 50% a year? + + PYTHONPATH=. uv run python -m research.lab.overnight_only # once: daily.pkl + PYTHONPATH=. uv run python -m research.lab.overnight_book + +Streams (daily, costs inside): + momentum overnight_recovery_top10 held all day (stream_portfolio streams.pkl) + overnight the same score's top 10 held close -> open only, 1 bp a side, + cash idle during the day earns nothing (overnight_only daily.pkl, + sleeve_score) + overnight_12_1 overnight momentum alone, same rules (reported only) + lowrisk, trend_v7 (trend_rider_v7_vt24, the live sleeve), trend (V10): streams.pkl + +Book: separate sleeves with equal base shares, drift, a month-start reset +when any share is > 25% off, 10 bp on capital moved +(managed_futures_book.band_blend). Sharpe is excess over the 3-month T-bill. + +Leverage (reported only): r_L = L x r - (L - 1) x (T-bill + 1%) / 252, the +L at which 2006-10..2026-09 CAGR reaches 50%, and MaxDD / worst year at it. + +All of 2006-2026 was inspected before, and the overnight result was seen +before this rule was written. This is a consistency gate for the forward test +(overnight_paper.py), not out-of-sample evidence. The rule is written to +prereg.json before the run; the script refuses to overwrite results.json. + +Result (2026-10-09): CONTINUE. Both candidates pass B1 and B2. + + book (1x) CAGR Sharpe halves MaxDD worst yr L for 50% MaxDD at L + three_sleeve_v7 (baseline) 18.9% 0.98 0.70 / 1.33 -37.2% -22.3% 3.58 -86.5% + three_sleeve_overnight 22.3% 1.38 1.16 / 1.64 -16.5% -7.0% 2.35 -36.7% + two_sleeve_overnight_trend 29.4% 1.39 1.12 / 1.69 -20.7% -13.8% 1.73 -34.8% + three_sleeve_overnight_12_1 24.9% 1.48 1.36 / 1.64 -15.7% -7.0% 2.05 -30.6% (reported) + + - Why it works: the overnight sleeve drops the all-day sleeve's -69% MaxDD + and 2008's -55% to -31% and -7%, at monthly corr 0.17 with trend_v7 + (all-day momentum: 0.32). 2008: book +3% vs -22%. + - Cost sensitivity (reported after the run), three_sleeve_overnight Sharpe + by cost per side: + 1 bp 1.38 + 2 bp 1.26 + 3 bp 1.13 + 4 bp 1.01 (halves 0.78 / 1.29), about the baseline. + The auction cost measured by the paper test decides how much of the gain + is real. + - Boundary: TrendRider before 2010 is synthetic 3x ETFs. The overnight + sleeve's idle daytime cash earns nothing here. Leverage above 2x + overnight needs portfolio margin or futures. +""" + +from __future__ import annotations + +import json +import os + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS +from research.lab.levered_book import stats, tbill +from research.lab.managed_futures_book import band_blend + +OUT = os.path.join(RESULTS, "overnight_book") +STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl") +OVERNIGHT = os.path.join(RESULTS, "overnight_only", "daily.pkl") +SPAN = ("2006-10-02", "2026-09-24") +HALVES = [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")] +BORROW_SPREAD = 0.01 + +BASELINE = ["momentum", "lowrisk", "trend_v7"] +CANDIDATES = { + "three_sleeve_overnight": ["overnight", "lowrisk", "trend_v7"], + "two_sleeve_overnight_trend": ["overnight", "trend_v7"], +} +REPORTED = { + "three_sleeve_v7": BASELINE, + "three_sleeve_overnight_v10": ["overnight", "lowrisk", "trend"], + "three_sleeve_overnight_12_1": ["overnight_12_1", "lowrisk", "trend_v7"], + "overnight_plus_momentum_lowrisk_trend": ["overnight", "momentum", "lowrisk", "trend_v7"], +} + +PREREG = { + "question": "Does replacing the all-day momentum sleeve with its overnight-only version raise the " + "three-sleeve book's Sharpe in both halves without a deeper drawdown?", + "span": SPAN, "halves": HALVES, "baseline": {"three_sleeve_v7": BASELINE}, + "candidates": CANDIDATES, "reported_only": REPORTED, + "blend": "managed_futures_book.band_blend (equal shares, 25% band, month-start reset, 10 bp)", + "gates_per_candidate": ["B1 excess Sharpe >= baseline + 0.15 in each half", + "B2 full-span MaxDD >= max(baseline MaxDD, -30%)"], + "decision": {"CONTINUE": "a candidate passes B1-B2 -> keep the overnight paper test running; after >= 20 " + "paper nights compare realized vs print returns, then decide on a live sleeve swap", + "STOP": "no candidate passes -> the overnight sleeve stays a paper experiment only"}, + "reported": "leverage L for 50%/yr CAGR (borrow at T-bill + 1%), MaxDD and worst year at L; yearly returns", + "data_status": "all periods previously inspected; overnight result seen before this rule", +} + + +def summary(r: pd.Series, rf: pd.Series) -> dict: + r = r.loc[SPAN[0]:SPAN[1]] + d = {k: float(v) for k, v in stats(r, rf).items()} + d["halves"] = [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES] + return d + + +def levered(r: pd.Series, rf: pd.Series, L: float) -> pd.Series: + return L * r - (L - 1) * (rf.reindex(r.index) + BORROW_SPREAD) / 252 + + +def leverage_for(r: pd.Series, rf: pd.Series, target: float = 0.50) -> float: + lo, hi = 1.0, 6.0 + if stats(levered(r, rf, hi), rf)["cagr"] < target: + return float("nan") + for _ in range(50): + mid = (lo + hi) / 2 + if stats(levered(r, rf, mid), rf)["cagr"] < target: + lo = mid + else: + hi = mid + return hi + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + R = pd.read_pickle(STREAMS) + O = pd.read_pickle(OVERNIGHT) + R["overnight"] = O["sleeve_score"].reindex(R.index) + R["overnight_12_1"] = O["overnight_12_1"].reindex(R.index) + R = R.loc[SPAN[0]:SPAN[1]] + rf = tbill(R.index) + + fmt = lambda k, d: (f"{k:40s} cagr {d['cagr']*100:5.1f}% xsh {d['sharpe']:5.2f} vol {d['vol']*100:4.1f}% " + f"mdd {d['mdd']*100:6.1f}% worst yr {d['worst_yr']*100:6.1f}% halves " + f"{d['halves'][0]:5.2f} {d['halves'][1]:5.2f}") + res = {"streams": {}, "books": {}, "leverage_for_50": {}} + print("=== streams") + for c in ["momentum", "overnight", "overnight_12_1", "lowrisk", "trend_v7", "trend"]: + res["streams"][c] = summary(R[c].dropna(), rf) + print(fmt(c, res["streams"][c])) + B = {} + print("=== books (1x)") + for name, cols in {**REPORTED, **CANDIDATES}.items(): + B[name], _ = band_blend(R[cols], rf) + res["books"][name] = summary(B[name], rf) + print(fmt(name, res["books"][name])) + + print("=== leverage for 50%/yr (borrow at T-bill + 1%)") + for name, r in B.items(): + L = leverage_for(r, rf) + d = {"L": L} + if np.isfinite(L): + s = stats(levered(r, rf, L), rf) + d.update({"mdd": float(s["mdd"]), "worst_yr": float(s["worst_yr"]), "vol": float(s["vol"]), + "halves_cagr": [float(stats(levered(r.loc[a:b], rf, L), rf)["cagr"]) for a, b in HALVES]}) + print(f" {name:40s} L {L:4.2f} vol {d['vol']*100:4.1f}% mdd {d['mdd']*100:6.1f}% worst yr " + f"{d['worst_yr']*100:6.1f}% halves cagr {d['halves_cagr'][0]*100:5.1f}% {d['halves_cagr'][1]*100:5.1f}%") + res["leverage_for_50"][name] = d + + m = (1 + R[["momentum", "overnight", "lowrisk", "trend_v7"]].fillna(0.0)).resample("ME").prod() - 1 + res["monthly_corr"] = m.corr().round(3).to_dict() + print("monthly corr:\n", m.corr().round(2).to_string()) + yr = pd.DataFrame({k: (1 + B[k]).groupby(B[k].index.year).prod() - 1 + for k in ["three_sleeve_v7", *CANDIDATES]}) + res["yearly"] = {k: {int(y): float(v) for y, v in yr[k].items()} for k in yr} + print("yearly %:\n", (yr * 100).round(1).to_string()) + + base = res["books"]["three_sleeve_v7"] + gates = {} + for k in CANDIDATES: + d = res["books"][k] + gates[k] = {"B1": all(d["halves"][i] >= base["halves"][i] + 0.15 for i in range(2)), + "B2": d["mdd"] >= max(base["mdd"], -0.30)} + passing = [k for k, g in gates.items() if all(g.values())] + decision = "CONTINUE" if passing else "STOP" + res["verdict"] = {"gates": gates, "passing": passing, "decision": decision} + print(f"gates {gates}\n-> {decision} {passing}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.DataFrame(B).to_pickle(os.path.join(OUT, "books.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"overnight_book:{k}", **{x: v for x, v in d.items() if x != "halves"}} + for k, d in res["books"].items()], + {"study": "overnight_book", "role": "preregistered consistency gate"}) + + +if __name__ == "__main__": + main() diff --git a/research/lab/overnight_book_mf.py b/research/lab/overnight_book_mf.py new file mode 100644 index 0000000..aa8a1f0 --- /dev/null +++ b/research/lab/overnight_book_mf.py @@ -0,0 +1,156 @@ +""" +At the leverage that takes the overnight books to 50% a year, does a +managed-futures (CTA) sleeve cut the drawdown enough to keep? + + PYTHONPATH=. uv run python -m research.lab.overnight_only # once: daily.pkl + PYTHONPATH=. uv run python -m research.lab.futures_trend # once: returns.pkl + PYTHONPATH=. uv run python -m research.lab.overnight_book_mf [--no-proxy] + +Streams: overnight (overnight-only momentum, 1 bp a side), lowrisk, trend_v7 +as in research/lab/overnight_book.py; mf as in +research/lab/managed_futures_book.py (DBMF from 2019-05-09, before it the +24-market futures TSMOM less 0.85%/yr; the proxy is not DBMF, monthly corr 0.64). + +Books: equal base shares, banded month-start reset (managed_futures_book.band_blend). +Each book is levered to the L at which its 2006-10..2026-09 CAGR is 50%: +r_L = L x r - (L - 1) x (T-bill + 1%) / 252 (overnight_book.levered). + +Pairs (candidate vs its base): + four_sleeve_overnight_mf overnight, lowrisk, trend_v7, mf vs three_sleeve_overnight + three_sleeve_overnight_trend_mf overnight, trend_v7, mf vs two_sleeve_overnight_trend + +All of 2006-2026 was inspected before (both the overnight books and the mf +sleeve). This is a consistency gate, not out-of-sample evidence. The rule is +written to prereg.json before the run; the script refuses to overwrite +results.json. + +Result (2026-10-09): KEEP. Both candidates fail M1 and M2. + + book 1x Sharpe halves L for 50% MaxDD at L worst yr at L + three_sleeve_overnight 1.38 1.16 / 1.64 2.35 -36.7% -18.6% + four_sleeve_overnight_mf 1.36 1.15 / 1.58 2.93 -37.6% -15.4% + two_sleeve_overnight_trend 1.39 1.12 / 1.69 1.73 -34.8% -24.3% + three_sleeve_overnight_trend_mf 1.36 1.10 / 1.63 2.41 -36.4% -20.8% + + - At 1x the mf sleeve trims MaxDD by 2-5 pp. It also costs 4-8 pp of + CAGR, so the book needs 0.6-0.7x more leverage for 50%, and the levered + MaxDD is unchanged. + - The overnight books no longer have the drawdown CTA insures against: + 2008 at L is -2% / +18% without mf. + - Their deepest drawdowns are 2008-06..11 (three-sleeve), 2014-07..2015-11 + (two-sleeve, a slow equity-momentum and trend slump) and 2020-02..03 + (with mf). Futures trend does not help in the first two. +""" + +from __future__ import annotations + +import argparse +import json +import os + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS +from research.lab.levered_book import stats, tbill +from research.lab.managed_futures_book import DBMF_FEE, DBMF_START, FUTURES, band_blend +from research.lab.overnight_book import HALVES, OVERNIGHT, SPAN, STREAMS, leverage_for, levered + +OUT = os.path.join(RESULTS, "overnight_book_mf") +PAIRS = { + "four_sleeve_overnight_mf": (["overnight", "lowrisk", "trend_v7", "mf"], "three_sleeve_overnight"), + "three_sleeve_overnight_trend_mf": (["overnight", "trend_v7", "mf"], "two_sleeve_overnight_trend"), +} +BASES = {"three_sleeve_overnight": ["overnight", "lowrisk", "trend_v7"], + "two_sleeve_overnight_trend": ["overnight", "trend_v7"]} + +PREREG = { + "question": "At the leverage giving 50%/yr, does adding a managed-futures sleeve to the overnight books " + "cut MaxDD by >= 5 pp without lowering the 1x Sharpe?", + "span": SPAN, "halves": HALVES, "pairs": {k: {"candidate": v[0], "base": v[1]} for k, v in PAIRS.items()}, + "bases": BASES, + "mf_stream": f"DBMF from {DBMF_START}; before it tsmom24 less {DBMF_FEE:.2%}/yr", + "leverage": "per book, L with full-span CAGR = 50%, borrowing at T-bill + 1%", + "gates_per_candidate": ["M1 MaxDD at its L >= base MaxDD at the base's L + 5 pp", + "M2 1x excess Sharpe >= base 1x Sharpe - 0.05 in each half"], + "decision": {"ADD_MF": "a candidate passes -> the book to forward-test includes a DBMF sleeve " + "(the one with the shallower levered MaxDD if both pass)", + "KEEP": "none passes -> no CTA sleeve; the overnight books stay as they are"}, + "reported": "worst year and vol at L, DBMF era (2019-05..) at L, yearly returns at L", + "data_status": "all periods previously inspected; synthetic 3x ETFs before 2010; mf proxy before 2019-05", +} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--no-proxy", action="store_true") + if not ap.parse_args().no_proxy: + from research.lab.fetch_us import _proxy_env + _proxy_env() + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + R = pd.read_pickle(STREAMS) + days = R.index + R["overnight"] = pd.read_pickle(OVERNIGHT)["sleeve_score"].reindex(days) + from research.lab.fetch_long_history import fetch_yahoo + fut = pd.read_pickle(FUTURES)["tsmom24"].reindex(days) + dbmf_px, _ = fetch_yahoo(["DBMF"], start="2019-01-01") + dbmf = dbmf_px["DBMF"].pct_change(fill_method=None).reindex(days).loc[DBMF_START:] + R["mf"] = pd.concat([fut.loc[:DBMF_START].iloc[:-1] - DBMF_FEE / 252, dbmf]).reindex(days) + R = R.loc[SPAN[0]:SPAN[1]] + rf = tbill(R.index) + + books = {**BASES, **{k: v[0] for k, v in PAIRS.items()}} + res = {"books": {}} + B = {} + for name, cols in books.items(): + r, _ = band_blend(R[cols], rf) + B[name] = r + one = stats(r, rf) + L = leverage_for(r, rf) + rl = levered(r, rf, L) + s = stats(rl, rf) + era = stats(rl.loc[DBMF_START:], rf) + d = {"x1": {"cagr": float(one["cagr"]), "sharpe": float(one["sharpe"]), "mdd": float(one["mdd"]), + "halves": [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES]}, + "L": float(L), "at_L": {"mdd": float(s["mdd"]), "worst_yr": float(s["worst_yr"]), "vol": float(s["vol"]), + "worst_3y": float(s["worst_3y"])}, + "dbmf_era_at_L": {"cagr": float(era["cagr"]), "sharpe": float(era["sharpe"]), "mdd": float(era["mdd"])}, + "yearly_at_L": {int(y): float(v) for y, v in ((1 + rl).groupby(rl.index.year).prod() - 1).items()}} + res["books"][name] = d + print(f"{name:34s} 1x cagr {d['x1']['cagr']*100:5.1f}% xsh {d['x1']['sharpe']:.2f} halves " + f"{d['x1']['halves'][0]:.2f} {d['x1']['halves'][1]:.2f} mdd {d['x1']['mdd']*100:6.1f}% | L {L:.2f}: " + f"mdd {s['mdd']*100:6.1f}% worst yr {s['worst_yr']*100:6.1f}% worst 3y/yr {s['worst_3y']*100:5.1f}% " + f"vol {s['vol']*100:4.1f}% | DBMF era at L cagr {era['cagr']*100:5.1f}% xsh {era['sharpe']:.2f} " + f"mdd {era['mdd']*100:6.1f}%", flush=True) + + yr = pd.DataFrame({k: res["books"][k]["yearly_at_L"] for k in books}) * 100 + print("yearly % at L:\n", yr.round(1).to_string()) + + gates = {} + for k, (_, base) in PAIRS.items(): + c, b = res["books"][k], res["books"][base] + gates[k] = {"M1": c["at_L"]["mdd"] >= b["at_L"]["mdd"] + 0.05, + "M2": all(c["x1"]["halves"][i] >= b["x1"]["halves"][i] - 0.05 for i in range(2))} + passing = [k for k, g in gates.items() if all(g.values())] + sel = max(passing, key=lambda k: res["books"][k]["at_L"]["mdd"]) if passing else None + decision = "ADD_MF" if passing else "KEEP" + res["verdict"] = {"gates": gates, "passing": passing, "selected": sel, "decision": decision} + print(f"gates {gates}\n-> {decision} {sel}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.DataFrame(B).to_pickle(os.path.join(OUT, "books.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"overnight_book_mf:{k}", "L": v["L"], **v["x1"], **{f"L_{x}": y for x, y in v["at_L"].items()}} + for k, v in res["books"].items()], + {"study": "overnight_book_mf", "role": "preregistered consistency gate"}) + + +if __name__ == "__main__": + main()