From 65b8db7a12b0d5d89d31add3600961fcdaf32312 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 9 Oct 2026 00:58:47 -0700 Subject: [PATCH] =?UTF-8?q?research:=2050%-target=20search=20=E2=80=94=20c?= =?UTF-8?q?onservative=20book,=20daily=20quick=20checks,=20Alpaca=20intrad?= =?UTF-8?q?ay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - managed_futures_book: three sleeves + managed futures (DBMF, tsmom24 proxy before 2019-05) cuts MaxDD -37% -> -27% but CAGR 16-19%, Sharpe unchanged (REJECT). - small_cap_reversal / turn_of_month / short_vol: preregistered quick checks, all STOP (net of spread costs, post-publication decay, post-2013 VRP 0.37). - fetch_alpaca: resume-safe Alpaca SIP intraday bars 2016+ (1-min ETFs, 5-min liquid S&P 1500), keys from .env and never logged. - intraday_momentum: the SPY noise-area strategy replicates (yearly corr 1.00 with the paper) but SPY Sharpe 0.99, ~0 after publication, and only QQQ of five other ETFs works (STOP). - orb_stocks_in_play: preregistered 5-min ORB replication (not yet run). Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 7 + research/lab/fetch_alpaca.py | 195 +++++++++++++++++++++ research/lab/intraday_momentum.py | 227 ++++++++++++++++++++++++ research/lab/managed_futures_book.py | 245 ++++++++++++++++++++++++++ research/lab/orb_stocks_in_play.py | 249 +++++++++++++++++++++++++++ research/lab/short_vol.py | 156 +++++++++++++++++ research/lab/small_cap_reversal.py | 212 +++++++++++++++++++++++ research/lab/turn_of_month.py | 155 +++++++++++++++++ 8 files changed, 1446 insertions(+) create mode 100644 research/lab/fetch_alpaca.py create mode 100644 research/lab/intraday_momentum.py create mode 100644 research/lab/managed_futures_book.py create mode 100644 research/lab/orb_stocks_in_play.py create mode 100644 research/lab/short_vol.py create mode 100644 research/lab/small_cap_reversal.py create mode 100644 research/lab/turn_of_month.py diff --git a/CLAUDE.md b/CLAUDE.md index 30bb5da..ce939d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,6 +84,13 @@ PYTHONPATH=. uv run python -m research.lab.universe_midcap # overni PYTHONPATH=. uv run --with numba python -m research.lab.stream_portfolio # risk-budgeted book of momentum/lowrisk/TrendRider/ETF TSMOM/BTC streams, 2006-26 (preregistered; REJECT 2026-10-08) PYTHONPATH=. uv run python -m research.lab.hedged_momentum # momentum with its beta hedged by index futures (quick check; STOP 2026-10-08) PYTHONPATH=. uv run python -m research.lab.futures_trend [--no-proxy] # 24-market futures TSMOM on Yahoo =F + data checks (quick check; Sharpe 0.44, DATA 2026-10-08) +PYTHONPATH=. uv run python -m research.lab.managed_futures_book [--no-proxy] # three sleeves + managed futures (tsmom24 proxy -> DBMF), 2006-26 (preregistered; REJECT 2026-10-08) +PYTHONPATH=. uv run python -m research.lab.small_cap_reversal # 5-day reversal on S&P 400/600, EW-hedged, spread costs (quick check; STOP 2026-10-08) +PYTHONPATH=. uv run python -m research.lab.turn_of_month # turn-of-month premium 1928-2026, 3x in the window (quick check; STOP 2026-10-08) +PYTHONPATH=. uv run python -m research.lab.short_vol [--no-proxy] # short VIX futures with a VIX/VIX3M filter, 2006-26 (quick check; STOP 2026-10-08) +PYTHONPATH=. uv run python -m research.lab.fetch_alpaca etfs|stocks|status # Alpaca free SIP intraday bars 2016+ (1-min ETFs, 5-min liquid S&P 1500; keys from .env, never logged) +PYTHONPATH=. uv run python -m research.lab.intraday_momentum # SPY noise-area intraday momentum replication + 5 ETFs (preregistered; STOP 2026-10-09) +PYTHONPATH=. uv run python -m research.lab.orb_stocks_in_play # 5-min opening-range breakout on the top-20 stocks in play (preregistered) uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs) uv run python -m research.lab.fetch_us_fundamentals --universe small # the same for S&P 600 (IJR) PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] [--universe small] # S&P 400 sleeves vs IJH; S&P 600 replication vs IJR (all REJECT 2026-10-08) diff --git a/research/lab/fetch_alpaca.py b/research/lab/fetch_alpaca.py new file mode 100644 index 0000000..6f45b56 --- /dev/null +++ b/research/lab/fetch_alpaca.py @@ -0,0 +1,195 @@ +""" +Intraday bars from Alpaca's market-data API (free Basic plan: full-market SIP +bars from 2016, all but the latest 15 minutes; delisted symbols included). + + PYTHONPATH=. uv run python -m research.lab.fetch_alpaca etfs # 1-min SPY QQQ IWM DIA TLT GLD + PYTHONPATH=. uv run python -m research.lab.fetch_alpaca stocks [--universe us|midsmall|all] # 5-min PIT ever-members + PYTHONPATH=. uv run python -m research.lab.fetch_alpaca status + +Keys come from ALPACA_API_KEY / ALPACA_API_SECRET in the environment or .env +and are never printed. Bars are split- and dividend-adjusted +(adjustment=all), regular session only (09:30-16:00 New York; a bar is +labeled by its start time), stored per symbol as +data/research/intraday/alpaca//.pkl, float32 columns +o h l c v vw n, index tz-aware New York. A symbol's file is written once +complete, so a rerun resumes. An empty result is recorded in _empty.json and +not retried. +""" + +from __future__ import annotations + +import argparse +import json +import os +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pandas as pd +import requests + +ROOT = os.path.join("data", "research", "intraday", "alpaca") +URL = "https://data.alpaca.markets/v2/stocks/{sym}/bars" +START, END = "2016-01-01", None +ETFS = ["SPY", "QQQ", "IWM", "DIA", "TLT", "GLD"] +MIN_INTERVAL = 60 / 180 # stay under the 200 requests/minute limit +WORKERS = 6 # requests take ~1-2 s; six threads reach the limit +NY = "America/New_York" + + +def _keys() -> dict[str, str]: + env = dict(os.environ) + if os.path.exists(".env"): + for line in open(".env"): + if "=" in line and not line.lstrip().startswith("#"): + k, v = line.split("=", 1) + env.setdefault(k.strip(), v.strip().strip("'\"")) + return {"APCA-API-KEY-ID": env["ALPACA_API_KEY"], "APCA-API-SECRET-KEY": env["ALPACA_API_SECRET"]} + + +class Client: + """Thread-safe: one session per thread, one shared request clock (<= 180/min).""" + + def __init__(self) -> None: + self.headers = _keys() + self.local = threading.local() + self.lock = threading.Lock() + self.next_slot = 0.0 + + def _session(self) -> requests.Session: + if not hasattr(self.local, "s"): + self.local.s = requests.Session() + self.local.s.headers.update(self.headers) + return self.local.s + + def _wait_turn(self) -> None: + with self.lock: + now = time.time() + slot = max(now, self.next_slot) + self.next_slot = slot + MIN_INTERVAL + if slot > now: + time.sleep(slot - now) + + def get(self, sym: str, params: dict) -> dict: + for attempt in range(8): + self._wait_turn() + try: + r = self._session().get(URL.format(sym=sym), params=params, timeout=60) + except requests.RequestException: + time.sleep(2 ** attempt) + continue + if r.status_code == 429 or r.status_code >= 500: + time.sleep(2 ** attempt) + continue + if r.status_code != 200: + raise RuntimeError(f"{sym}: HTTP {r.status_code} {r.text[:200]}") + return r.json() + raise RuntimeError(f"{sym}: retries exhausted") + + +def fetch_symbol(cl: Client, sym: str, timeframe: str, start: str, end: str) -> pd.DataFrame: + params = {"timeframe": timeframe, "start": start, "end": end, "feed": "sip", "limit": 10000, + "adjustment": "all"} + parts = [] + while True: + j = cl.get(sym.replace("-", "."), params) + if j.get("bars"): + df = pd.DataFrame(j["bars"]) + df.index = pd.DatetimeIndex(pd.to_datetime(df.pop("t"), utc=True)).tz_convert(NY) + hm = df.index.hour * 60 + df.index.minute + df = df[(hm >= 9 * 60 + 30) & (hm < 16 * 60)] + parts.append(df.reindex(columns=["o", "h", "l", "c", "v", "vw", "n"]).astype(np.float32)) + tok = j.get("next_page_token") + if not tok: + break + params["page_token"] = tok + return pd.concat(parts) if parts else pd.DataFrame() + + +def run(symbols: list[str], timeframe: str) -> None: + out = os.path.join(ROOT, timeframe) + os.makedirs(out, exist_ok=True) + empty_path = os.path.join(out, "_empty.json") + empty = set(json.load(open(empty_path))) if os.path.exists(empty_path) else set() + end = (pd.Timestamp.now(tz="UTC") - pd.Timedelta(minutes=20)).strftime("%Y-%m-%dT%H:%M:%SZ") + cl = Client() + todo = [s for s in symbols if s not in empty and not os.path.exists(os.path.join(out, f"{s}.pkl"))] + print(f"{timeframe}: {len(symbols)} symbols, {len(todo)} to fetch", flush=True) + t0 = time.time() + lock = threading.Lock() + done = [0] + + def one(s: str) -> None: + try: + df = fetch_symbol(cl, s, timeframe, START, end) + except RuntimeError as e: + print(f" {s}: {e}", flush=True) + return + if not df.empty: + tmp = os.path.join(out, f".{s}.pkl.tmp") + df.to_pickle(tmp) + os.replace(tmp, os.path.join(out, f"{s}.pkl")) + with lock: + if df.empty: + empty.add(s) + json.dump(sorted(empty), open(empty_path, "w")) + done[0] += 1 + i = done[0] + if i % 10 == 0 or i == len(todo): + print(f" {i}/{len(todo)} {s} rows {len(df)} elapsed {(time.time()-t0)/60:.1f} min", flush=True) + + with ThreadPoolExecutor(max_workers=WORKERS) as ex: + list(ex.map(one, todo)) + + +def _liquid(p) -> pd.DataFrame: + """Member days that pass the day-trading filters of research/lab/orb_stocks_in_play.py + (14-day mean volume >= 1M shares, close > $5, 14-day mean range > $0.50) on daily bars.""" + vol = p.volume.rolling(14, min_periods=10).mean() + rng = (p.high - p.low).rolling(14, min_periods=10).mean() + return p.member & (vol >= 1e6) & (p.close > 5) & (rng > 0.5) + + +def universe(name: str, liquid: bool = True) -> list[str]: + """PIT members at any time since 2016 (Yahoo spelling); with ``liquid``, only names + that passed the day-trading filters on at least one member day.""" + syms: set[str] = set() + panels = [] + if name in ("us", "all"): + from research.lab.factor_scan import load + panels.append(load("us")[0]) + if name in ("midsmall", "all"): + from research.lab.panel_mid import load_us_mid + panels.append(load_us_mid("midsmall")) + for p in panels: + m = (_liquid(p) if liquid else p.member).loc[START:] + syms |= set(m.columns[m.any()]) + return sorted(s for s in syms if s.replace("-", "").replace(".", "").isalnum()) + + +def status() -> None: + for tf in sorted(os.listdir(ROOT)) if os.path.exists(ROOT) else []: + d = os.path.join(ROOT, tf) + files = [f for f in os.listdir(d) if f.endswith(".pkl")] + empty = json.load(open(os.path.join(d, "_empty.json"))) if os.path.exists(os.path.join(d, "_empty.json")) else [] + size = sum(os.path.getsize(os.path.join(d, f)) for f in files) / 1e9 + print(f"{tf}: {len(files)} symbols, {len(empty)} empty, {size:.2f} GB") + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("what", choices=["etfs", "stocks", "status"]) + ap.add_argument("--universe", default="all", choices=["us", "midsmall", "all"]) + ap.add_argument("--all-members", action="store_true", help="skip the liquidity pre-filter") + a = ap.parse_args() + if a.what == "status": + status() + elif a.what == "etfs": + run(ETFS, "1Min") + else: + run(universe(a.universe, liquid=not a.all_members), "5Min") + + +if __name__ == "__main__": + main() diff --git a/research/lab/intraday_momentum.py b/research/lab/intraday_momentum.py new file mode 100644 index 0000000..1d81a07 --- /dev/null +++ b/research/lab/intraday_momentum.py @@ -0,0 +1,227 @@ +""" +Replication of Zarattini, Aziz & Barbon, "Beat the Market: An Effective +Intraday Momentum Strategy for S&P500 ETF (SPY)" (SSRN 4824172, first posted +2024-05), on Alpaca 1-minute SIP bars 2016-2026, then the same rules on five +other ETFs. Intraday only: flat at every close. + + PYTHONPATH=. uv run python -m research.lab.fetch_alpaca etfs # once + PYTHONPATH=. uv run python -m research.lab.intraday_momentum + +Rules (from the paper's text; "dynamic" sizing, current band + VWAP stop): + move(d, m) |close(d, m) / open(d, 09:30) - 1|, m = minute of day + sigma(d, m) mean of move over the previous 14 days at the same minute + bands UB = max(open_d, close_{d-1}) x (1 + sigma); LB = min(open_d, close_{d-1}) x (1 - sigma) + decisions only at 10:00, 10:30, ..., 15:30, on the close of the minute that ends there: + long if price > UB and > VWAP, short if price < LB and < VWAP, else flat + (the band + VWAP trailing stop), filled at the open of the next minute + exit at the day's last close (16:00, or the early close) + size shares = equity_{d-1} x min(4, 2% / sd of the last 14 daily returns) / open_d + costs $0.0035 commission + $0.001 slippage per share traded +The account's cash earns nothing here, so Sharpe is on the strategy's own +returns (it holds no overnight position). + +Fidelity check: the paper's yearly returns (its Table 9, dynamic sizing) for +2016-2023 are a reference for the implementation, not for the edge. + +The rule is written to prereg.json before the run; the script refuses to +overwrite results.json. 2016-2024-04 lies inside the paper's own sample +(in-sample for the authors); 2024-05 onward is after publication. + +Result (2026-10-09): STOP. F1 and E2 pass; E1 and E3 fail. + - Fidelity: our SPY yearly returns 2016-23 match the paper's (corr 1.00, + mean |diff| 2.3 pp). + - SPY net 14.1% / Sharpe 0.99 / MaxDD -26%, halves 0.74 / 1.20. + After publication: Sharpe 0.06 (2025 -3.6%, 2026 YTD -11.1%). + - Same rules, untuned, on the other five ETFs: + - QQQ 21.5% / 1.42 (halves 1.26 / 1.57, post-publication 1.24). + - IWM 0.07, DIA 0.03, TLT -0.36, GLD 0.02. + - Equal-capital 6-ETF book: 5.2% / Sharpe 0.65. +Interpretation: the edge is not a general intraday-momentum property of +liquid ETFs. QQQ is one of six tries; read alone it is a hypothesis, and it +needs untouched data (QQQ before 2016) before any use. +""" + +from __future__ import annotations + +import json +import os + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS + +OUT = os.path.join(RESULTS, "intraday_momentum") +BARS = os.path.join("data", "research", "intraday", "alpaca", "1Min") +ETFS = ["SPY", "QQQ", "IWM", "DIA", "TLT", "GLD"] +SPAN = ("2016-02-01", "2026-10-08") +HALVES = [("2016-02-01", "2020-12-31"), ("2021-01-01", "2026-10-08")] +POST_PUB = ("2024-05-01", "2026-10-08") +LOOK, VOL_TARGET, MAX_LEV, PER_SHARE = 14, 0.02, 4.0, 0.0045 +CHECKS = list(range(30, 361, 30)) # minutes after 09:30: 10:00 .. 15:30 +PAPER_YEARLY = {2016: -12.8, 2017: -6.9, 2018: 61.1, 2019: 6.9, 2020: 26.8, 2021: 34.8, 2022: 24.4, 2023: 37.2} + +PREREG = { + "question": "Does the SPY noise-area intraday momentum strategy replicate on Alpaca data, survive " + "publication, and generalize to other ETFs?", + "rules": "paper text: 14-day noise area with gap adjustment, HH:00/HH:30 decisions, band+VWAP stop, " + "flat at close, size min(4, 2%/sd14), $0.0045/share", + "fidelity": "F1 corr of our SPY yearly returns with the paper's 2016-2023 >= 0.8 and mean |diff| <= 8 pp " + "(otherwise FIX the implementation; no edge verdict)", + "continue_if_all": [ + "E1 SPY net Sharpe >= 1.0 over 2016-02..2026-10", + "E2 SPY net Sharpe >= 0 after publication (2024-05..2026-10)", + "E3 net Sharpe >= 0.5 over 2016-26 on at least 3 of QQQ IWM DIA TLT GLD (same rules, untuned)", + ], + "reported": ["halves 2016-20 / 2021-26", "equal-capital 6-ETF book", + "book gate for paper trading: net Sharpe >= 1.5 in both halves"], + "decision": {"CONTINUE": "build the multi-ETF intraday book, stress costs, and paper-trade it on Alpaca", + "STOP": "drop ETF intraday momentum", "FIX": "implementation does not match the paper"}, +} + + +def day_matrices(df: pd.DataFrame) -> dict: + """(days x 390) minute matrices; columns are minutes after 09:30.""" + idx = df.index + minute = (idx.hour * 60 + idx.minute - 570).to_numpy() + dates = idx.tz_localize(None).normalize() + days = pd.DatetimeIndex(sorted(set(dates))) + row = days.get_indexer(dates) + D = len(days) + M = {} + for k in ("o", "c", "v", "vw"): + a = np.full((D, 390), np.nan) + a[row, minute] = df[k].to_numpy(dtype=float) + M[k] = a + last = np.full(D, -1) + np.maximum.at(last, row, minute) + first_open = pd.Series(df["o"].to_numpy(dtype=float), index=row).groupby(level=0).first().to_numpy() + c = pd.DataFrame(M["c"]).ffill(axis=1).to_numpy() + cols = np.arange(390)[None, :] + c[cols > last[:, None]] = np.nan # nothing after an early close + o = np.where(np.isnan(M["o"]), np.c_[np.full(D, np.nan), c[:, :-1]], M["o"]) + v = np.nan_to_num(M["v"]) + pv = np.nan_to_num(M["vw"]) * v + with np.errstate(invalid="ignore", divide="ignore"): + vwap = np.cumsum(pv, axis=1) / np.cumsum(v, axis=1) + day_close = c[np.arange(D), last] + return {"days": days, "c": c, "o": o, "vwap": vwap, "open": first_open, "last": last, "close": day_close} + + +def backtest(M: dict) -> pd.DataFrame: + c, o, vwap, op, last, close = M["c"], M["o"], M["vwap"], M["open"], M["last"], M["close"] + D = len(op) + move = np.abs(c / op[:, None] - 1) + sigma = pd.DataFrame(move).rolling(LOOK, min_periods=10).mean().shift(1).to_numpy() + prev_close = np.r_[np.nan, close[:-1]] + ret_d = pd.Series(close / prev_close - 1) + sd = ret_d.rolling(LOOK, min_periods=LOOK).std().shift(1).to_numpy() # through d-1 + lev = np.minimum(MAX_LEV, VOL_TARGET / sd) + hi = np.fmax(op, prev_close) + lo = np.fmin(op, prev_close) + gross, net, trades, inmkt = np.zeros(D), np.zeros(D), np.zeros(D), np.zeros(D) + for d in range(D): + if not np.isfinite(lev[d]) or not np.isfinite(hi[d]): + continue + pos, entry, pnl, traded = 0, 0.0, 0.0, 0.0 + for t in CHECKS: + if t > last[d] or not np.isfinite(sigma[d, t - 1]): + break + p, s, vw = c[d, t - 1], sigma[d, t - 1], vwap[d, t - 1] + ub, lb = hi[d] * (1 + s), lo[d] * (1 - s) + new = 1 if (p > ub and p > vw) else (-1 if (p < lb and p < vw) else 0) + if new != pos: + px = o[d, t] + if pos != 0: + pnl += pos * (px - entry) + traded += abs(new - pos) + pos, entry = new, px + inmkt[d] = 1 + if pos != 0: + pnl += pos * (close[d] - entry) + traded += abs(pos) + gross[d] = lev[d] * pnl / op[d] + net[d] = gross[d] - lev[d] * traded * PER_SHARE / op[d] + trades[d] = traded + return pd.DataFrame({"gross": gross, "net": net, "lev": lev, "traded": trades, "in_market": inmkt}, + index=M["days"]) + + +def describe(r: pd.Series) -> dict: + r = r.dropna() + eq = (1 + r).cumprod() + yrs = len(r) / 252 + return {"cagr": float(eq.iloc[-1] ** (1 / yrs) - 1) if yrs > 0 else float("nan"), + "vol": float(r.std() * np.sqrt(252)), + "sharpe": float(r.mean() / r.std() * np.sqrt(252)) if r.std() > 0 else float("nan"), + "mdd": float((eq / eq.cummax() - 1).min())} + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + R, res = {}, {"etfs": {}} + for sym in ETFS: + bt = backtest(day_matrices(pd.read_pickle(os.path.join(BARS, f"{sym}.pkl")))) + R[sym] = bt + d = {"full": describe(bt["net"].loc[SPAN[0]:SPAN[1]]), + "gross": describe(bt["gross"].loc[SPAN[0]:SPAN[1]]), + "halves": [describe(bt["net"].loc[a:b]) for a, b in HALVES], + "post_pub": describe(bt["net"].loc[POST_PUB[0]:POST_PUB[1]]), + "days_in_market": float(bt["in_market"].loc[SPAN[0]:].mean()), + "avg_lev": float(bt["lev"].loc[SPAN[0]:].mean()), + "shares_traded_per_day_x_lev": float((bt["traded"] * bt["lev"]).loc[SPAN[0]:].mean())} + yr = (1 + bt["net"].loc[SPAN[0]:SPAN[1]]).groupby(bt.loc[SPAN[0]:SPAN[1]].index.year).prod() - 1 + d["yearly"] = {int(k): float(v) for k, v in yr.items()} + res["etfs"][sym] = d + f = d["full"] + print(f"{sym:4s} net cagr {f['cagr']*100:5.1f}% vol {f['vol']*100:4.1f}% sh {f['sharpe']:5.2f} " + f"mdd {f['mdd']*100:6.1f}% | gross sh {d['gross']['sharpe']:5.2f} | halves " + f"{d['halves'][0]['sharpe']:5.2f} {d['halves'][1]['sharpe']:5.2f} | post-pub sh " + f"{d['post_pub']['sharpe']:5.2f} cagr {d['post_pub']['cagr']*100:5.1f}% | in mkt " + f"{d['days_in_market']:.2f} lev {d['avg_lev']:.2f}", flush=True) + print(" yearly %:", {k: round(v * 100, 1) for k, v in d["yearly"].items()}) + + ours = pd.Series({y: res["etfs"]["SPY"]["yearly"].get(y, np.nan) * 100 for y in PAPER_YEARLY}) + paper = pd.Series(PAPER_YEARLY) + fid = {"corr": float(ours.corr(paper)), "mean_abs_diff_pp": float((ours - paper).abs().mean()), + "ours": ours.round(1).to_dict(), "paper": PAPER_YEARLY} + res["fidelity"] = fid + print(f"fidelity vs paper 2016-23: corr {fid['corr']:.2f}, mean |diff| {fid['mean_abs_diff_pp']:.1f} pp") + + book = pd.DataFrame({s: R[s]["net"] for s in ETFS}).loc[SPAN[0]:SPAN[1]].fillna(0.0).mean(axis=1) + res["book_equal6"] = {"full": describe(book), "halves": [describe(book.loc[a:b]) for a, b in HALVES], + "post_pub": describe(book.loc[POST_PUB[0]:])} + corr = pd.DataFrame({s: R[s]["net"] for s in ETFS}).loc[SPAN[0]:].corr().round(2) + res["daily_corr"] = corr.to_dict() + b = res["book_equal6"] + print(f"book (1/6 each) net cagr {b['full']['cagr']*100:5.1f}% sh {b['full']['sharpe']:5.2f} mdd " + f"{b['full']['mdd']*100:6.1f}% halves {b['halves'][0]['sharpe']:5.2f} {b['halves'][1]['sharpe']:5.2f} " + f"post-pub {b['post_pub']['sharpe']:5.2f}") + print("daily corr:\n", corr.to_string()) + + spy = res["etfs"]["SPY"] + others = [s for s in ETFS if s != "SPY" and res["etfs"][s]["full"]["sharpe"] >= 0.5] + f1 = fid["corr"] >= 0.8 and fid["mean_abs_diff_pp"] <= 8 + checks = {"F1": f1, "E1": spy["full"]["sharpe"] >= 1.0, "E2": spy["post_pub"]["sharpe"] >= 0, + "E3": len(others) >= 3} + decision = "FIX" if not f1 else ("CONTINUE" if all(checks.values()) else "STOP") + res["verdict"] = {"checks": checks, "others_passing": others, "decision": decision, + "book_paper_trading_gate": all(h["sharpe"] >= 1.5 for h in b["halves"])} + print(f"checks {checks} others {others} -> {decision}; book gate {res['verdict']['book_paper_trading_gate']}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.concat({s: R[s] for s in ETFS}, axis=1).to_pickle(os.path.join(OUT, "daily.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"intraday_momentum:{s}", **res["etfs"][s]["full"]} for s in ETFS], + {"study": "intraday_momentum", "role": "preregistered replication"}) + + +if __name__ == "__main__": + main() diff --git a/research/lab/managed_futures_book.py b/research/lab/managed_futures_book.py new file mode 100644 index 0000000..49b6c3a --- /dev/null +++ b/research/lab/managed_futures_book.py @@ -0,0 +1,245 @@ +""" +The conservative US book: does adding a managed-futures (futures trend) +sleeve to the live three-sleeve book give a steadier book with 20%+ a year +and a drawdown no worse than -30%, without lowering the Sharpe in either half +of 2006-2026? + + PYTHONPATH=. uv run --with numba python -m research.lab.stream_portfolio # once: streams.pkl + PYTHONPATH=. uv run python -m research.lab.futures_trend # once: returns.pkl + PYTHONPATH=. uv run python -m research.lab.managed_futures_book [--no-proxy] + +Streams (daily, costs inside; research/lab/stream_portfolio.py streams.pkl): + momentum overnight_recovery_top10, PIT S&P 500, lab engine + lowrisk lowrisk_top20 (ivol + low beta), lab engine + trend_v7 trend_rider_v7_vt24 on synthetic 3x ETFs, "lag" fills (the live sleeve) + trend_v7_vt36 trend_rider_v7_vt36, same engine (built here) + trend trend_rider_v10, same engine (reported only) + mf managed futures, the sleeve live trading can hold in a cash account: + DBMF (iMGP DBi Managed Futures ETF, replicates the SG CTA index with + futures; adjusted closes) from its first full day 2019-05-09; before + that the 24-market futures TSMOM of research/lab/futures_trend.py + (tsmom24) less DBMF's 0.85%/yr fee. The proxy is not DBMF: their + monthly correlation over 2019-26 is 0.64. + +Book: sleeves held as separate accounts in one book, as +strategies/sleeve_portfolio.py does live: shares drift with each sleeve's +return and are reset to the base shares at a month start when any share is +more than 25% (relative) off its base; 10 bp on the capital moved. + +All of 2006-2026 was inspected in earlier studies (TrendRider was tuned on +2010-2026; the futures sleeve's blend was explored in futures_trend.py), so +this is a consistency gate, not out-of-sample evidence. The forward test is +the evidence. The rule is written to prereg.json before the run, and the +script refuses to overwrite results.json. + +Result (2026-10-08): REJECT. Every candidate passes G2 and fails G1 and G3. +Excess Sharpe, 2006-10..2026-09, halves 2006-16 / 2017-26: + + | book | CAGR | Sharpe | halves | MaxDD | worst yr | + |------------------------------|-------|--------|-------------|-------|----------| + | three_sleeve_v7 (baseline) | 18.9% | 0.98 | 0.697/1.332 | -37% | -22.3% | + | four_sleeve_v7 | 16.3% | 1.01 | 0.763/1.301 | -27% | -11.3% | + | four_sleeve_v7_vt36 | 17.2% | 1.00 | 0.723/1.325 | -29% | -12.0% | + | three_sleeve_no_lowrisk | 19.1% | 1.01 | 0.721/1.329 | -28% | -8.9% | + | four_sleeve_v10 (reported) | 18.0% | 1.06 | 0.759/1.421 | -28% | -10.4% | + | mf stream (proxy -> DBMF) | 6.2% | 0.37 | 0.29/0.50 | -35% | -25.4% | + + - G1 misses in 2017-26 by 0.003-0.03 Sharpe: a tie, not an improvement. + - G3: CAGR 16-19% is short of 20%. + - The futures sleeve cuts MaxDD by about a quarter (-37% -> -27%) and + halves the worst year (2008: -22% -> -9..-12%). It costs 0-2.6 pp of + CAGR and leaves the Sharpe where it was (monthly corr with the other + sleeves 0.01-0.24). + - DBMF era, 2019-05..2026-09 (reported only): baseline 25.7% / 1.27 / + -18%; four_sleeve_v7 21.9% / 1.28 / -15%; three_sleeve_no_lowrisk + 27.4% / 1.32 / -18%. +Interpretation: in a 1x cash account these streams give about 16-19% a year +at Sharpe ~1.0 over 2006-26. The "Sharpe 1.2, 20-30%" range came from +equal5 (with BTC) and the 2017-26 half, not from this book. +Decision: register nothing by this rule. +""" + +from __future__ import annotations + +import argparse +import json +import os +import pickle + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS +from research.lab.levered_book import stats, tbill + +OUT = os.path.join(RESULTS, "managed_futures_book") +STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl") +FUTURES = os.path.join(RESULTS, "futures_trend", "returns.pkl") +SPAN = ("2006-10-02", "2026-09-24") +HALVES = [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")] +DBMF_START = "2019-05-09" +DBMF_FEE = 0.0085 +BAND, COST = 0.25, 0.001 + +CANDIDATES = { + "four_sleeve_v7": ["momentum", "lowrisk", "trend_v7", "mf"], + "four_sleeve_v7_vt36": ["momentum", "lowrisk", "trend_v7_vt36", "mf"], + "three_sleeve_no_lowrisk": ["momentum", "trend_v7", "mf"], +} +REPORTED = { + "three_sleeve_v7": ["momentum", "lowrisk", "trend_v7"], + "four_sleeve_v10": ["momentum", "lowrisk", "trend", "mf"], + "four_sleeve_v7_tsmom_only": ["momentum", "lowrisk", "trend_v7", "tsmom24"], + "four_sleeve_v7_etf_xtrend": ["momentum", "lowrisk", "trend_v7", "xtrend"], +} + +PREREG = { + "question": "Is momentum + lowrisk + TrendRider V7 + managed futures (equal shares) a steadier book " + "with CAGR >= 20% and MaxDD >= -30%, no worse than the live three-sleeve book in either half?", + "span": SPAN, "halves": HALVES, + "candidates": CANDIDATES, + "baseline": "three_sleeve_v7 (live us_three_sleeve: 1/3 momentum, lowrisk, trend_rider_v7_vt24), same blend", + "mf_stream": f"DBMF from {DBMF_START}; before it tsmom24 (futures_trend.py) less {DBMF_FEE:.2%}/yr", + "blend": f"equal base shares, drift, reset at a month start when any share is > {BAND:.0%} off; " + f"{COST*1e4:.0f} bp on capital moved", + "sharpe": "excess over the 3-month T-bill, daily, annualized", + "gates_per_candidate": [ + "G1 Sharpe >= three_sleeve_v7 Sharpe in each half", + "G2 full-span MaxDD >= -30%", + "G3 full-span CAGR >= 20%", + ], + "selection": "among candidates passing G1-G3, the highest min-half Sharpe -> ACCEPT; else among those " + "passing G1-G2, the highest CAGR -> PARTIAL (meets the risk goal, not the return goal); " + "else REJECT", + "decision": {"ACCEPT": "register the selected book and forward-test it as the conservative book", + "PARTIAL": "register it for forward testing, labeled as below the 20% return goal", + "REJECT": "register nothing; report which gate failed"}, + "reported_not_decisive": list(REPORTED) + ["DBMF-era (2019-05..) stats", "yearly returns", + "monthly correlations"], + "boundary": "all periods previously inspected; synthetic 3x ETFs before 2010; the mf proxy before " + "2019-05 is a different strategy from DBMF", +} + + +def band_blend(R: pd.DataFrame, rf: pd.Series) -> tuple[pd.Series, pd.DataFrame]: + """Book return of equal-share sleeves with drift and a banded month-start reset. + + A sleeve with no return yet (before its first observation) holds T-bills.""" + X = R.copy() + for c in X: + X[c] = X[c].fillna(rf.reindex(X.index).fillna(0.0) / 252) + n = X.shape[1] + base = np.full(n, 1.0 / n) + s = base.copy() + months = X.index.to_period("M") + out, shares = np.empty(len(X)), np.empty((len(X), n)) + vals = X.to_numpy() + for k in range(len(X)): + cost = 0.0 + if k > 0 and months[k] != months[k - 1] and np.max(np.abs(s / base - 1)) > BAND: + cost = np.abs(s - base).sum() * COST + s = base.copy() + shares[k] = s + g = s * (1 + vals[k]) + out[k] = g.sum() - 1 - cost + s = g / g.sum() + return pd.Series(out, index=X.index), pd.DataFrame(shares, index=X.index, columns=X.columns) + + +def summary(r: pd.Series, rf: pd.Series) -> dict: + r = r.loc[SPAN[0]:SPAN[1]] + d = {k: float(v) for k, v in stats(r, rf).items()} + d["halves"] = [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES] + return d + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--no-proxy", action="store_true") + if not ap.parse_args().no_proxy: + from research.lab.fetch_us import _proxy_env + _proxy_env() + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + R = pd.read_pickle(STREAMS) + days = R.index + rf = tbill(days) + from research.lab.fetch_long_history import fetch_yahoo + from research.lab.letf_history import run as run_letf + from trader import STRATEGY_REGISTRY + with open(os.path.join("data", "research", "letf_history.pkl"), "rb") as fh: + syn = pickle.load(fh)["close"].loc["1989-01-01":] + R["trend_v7_vt36"] = run_letf(STRATEGY_REGISTRY["trend_rider_v7_vt36"](), syn, "lag").pct_change().reindex(days) + fut = pd.read_pickle(FUTURES)["tsmom24"].reindex(days) + dbmf_px, _ = fetch_yahoo(["DBMF"], start="2019-01-01") + dbmf = dbmf_px["DBMF"].pct_change(fill_method=None).reindex(days).loc[DBMF_START:] + R["tsmom24"] = fut + R["mf"] = pd.concat([fut.loc[:DBMF_START].iloc[:-1] - DBMF_FEE / 252, dbmf]).reindex(days) + + B, avg_shares = {}, {} + for name, cols in {**CANDIDATES, **REPORTED}.items(): + B[name], sh = band_blend(R.loc[SPAN[0]:SPAN[1], cols], rf) + avg_shares[name] = sh.mean().round(3).to_dict() + + res = {"streams": {}, "books": {}} + fmt = lambda k, d: (f"{k:28s} cagr {d['cagr']*100:5.1f}% xsh {d['sharpe']:5.2f} vol {d['vol']*100:4.1f}% " + f"mdd {d['mdd']*100:6.1f}% worst yr {d['worst_yr']*100:6.1f}% halves " + f"{d['halves'][0]:5.2f} {d['halves'][1]:5.2f}") + print("=== streams") + for c in ["momentum", "lowrisk", "trend_v7", "trend_v7_vt36", "trend", "mf", "tsmom24", "xtrend"]: + res["streams"][c] = summary(R[c].dropna(), rf) + print(fmt(c, res["streams"][c])) + print("=== books") + for k, r in B.items(): + res["books"][k] = summary(r, rf) + print(fmt(k, res["books"][k])) + + # DBMF era: the real instrument, reported only + res["dbmf_era"] = {} + print(f"=== DBMF era {DBMF_START}..{SPAN[1]} (reported only)") + for k in ["three_sleeve_v7", *CANDIDATES]: + r = B[k].loc[DBMF_START:] + res["dbmf_era"][k] = {x: float(v) for x, v in stats(r, rf).items()} + print(f" {k:28s} cagr {res['dbmf_era'][k]['cagr']*100:5.1f}% xsh {res['dbmf_era'][k]['sharpe']:5.2f} " + f"mdd {res['dbmf_era'][k]['mdd']*100:6.1f}%") + m = (1 + R.loc[SPAN[0]:SPAN[1], ["momentum", "lowrisk", "trend_v7", "mf"]].fillna(0.0)).resample("ME").prod() - 1 + print("monthly corr:\n", m.corr().round(2).to_string()) + res["monthly_corr"] = m.corr().round(3).to_dict() + yr = pd.DataFrame({k: (1 + B[k]).groupby(B[k].index.year).prod() - 1 + for k in ["three_sleeve_v7", *CANDIDATES]}) + print("yearly %:\n", (yr * 100).round(1).to_string()) + res["yearly"] = {k: {int(y): float(v) for y, v in yr[k].items()} for k in yr} + res["avg_shares"] = avg_shares + + base = res["books"]["three_sleeve_v7"]["halves"] + gates = {} + for k in CANDIDATES: + d = res["books"][k] + gates[k] = {"G1": all(d["halves"][i] >= base[i] for i in range(2)), + "G2": d["mdd"] >= -0.30, "G3": d["cagr"] >= 0.20} + full = [k for k, g in gates.items() if all(g.values())] + risk = [k for k, g in gates.items() if g["G1"] and g["G2"]] + if full: + sel, decision = max(full, key=lambda k: min(res["books"][k]["halves"])), "ACCEPT" + elif risk: + sel, decision = max(risk, key=lambda k: res["books"][k]["cagr"]), "PARTIAL" + else: + sel, decision = None, "REJECT" + res["verdict"] = {"gates": gates, "selected": sel, "decision": decision} + print(f"gates {gates}\n-> {decision} {sel}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.DataFrame(B).to_pickle(os.path.join(OUT, "books.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"managed_futures_book:{k}", **v} for k, v in res["books"].items()], + {"study": "managed_futures_book", "role": "preregistered gate; all periods previously inspected"}) + + +if __name__ == "__main__": + main() diff --git a/research/lab/orb_stocks_in_play.py b/research/lab/orb_stocks_in_play.py new file mode 100644 index 0000000..4bf0309 --- /dev/null +++ b/research/lab/orb_stocks_in_play.py @@ -0,0 +1,249 @@ +""" +Replication of Zarattini, Barbon & Aziz, "A Profitable Day Trading Strategy +For The U.S. Equity Market" (SSRN 4729284, 2024-02): the 5-minute opening +range breakout on the 20 "stocks in play" of the day, on Alpaca 5-minute SIP +bars 2016-2026. Intraday only: flat at every close. + + PYTHONPATH=. uv run python -m research.lab.fetch_alpaca stocks # once (hours) + PYTHONPATH=. uv run python -m research.lab.orb_stocks_in_play + +Rules (from the paper's text): + filters open > $5; mean daily volume of the previous 14 days >= 1,000,000 + shares; 14-day ATR > $0.50; relative volume >= 1, where relative + volume = volume of the 09:30-09:35 bar / its mean over the previous + 14 days; then the 20 highest relative volumes + direction long if the first 5-minute bar closes above its open, short if + below, none for a doji + entry stop order at the opening-range high (long) / low (short) from 09:35 + stop loss 10% of the 14-day ATR from the fill + exit stop, or the day's last close + size loss at the stop = 1% of equity, each position's notional capped at + 4 x equity / 20 (the paper's 4x leverage limit) + costs $0.0035 per share per side (the paper); stress: + 5 bp per side +Deviations, fixed before the run: + - Universe: PIT S&P 500/400/600 members on the day, not the paper's ~7,000 + US stocks. Most "in play" small caps outside the S&P 1500 are missing. + - 5-minute bars: an entry at a gap-through bar fills at that bar's open. If + the entry bar also reaches the stop, the trade is counted as stopped out + (conservative). A stop gapped through fills at the next bar's open. + - Daily volume, ATR and closes come from the regular-session bars. + +The rule is written to prereg.json before the run; the script refuses to +overwrite results.json. 2016-2023 is the paper's own sample; 2024-01 onward +is after it. +""" + +from __future__ import annotations + +import json +import os + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS + +OUT = os.path.join(RESULTS, "orb_stocks_in_play") +BARS = os.path.join("data", "research", "intraday", "alpaca", "5Min") +SPAN = ("2016-02-01", "2026-10-08") +PAPER = ("2016-02-01", "2023-12-31") +POST = ("2024-01-01", "2026-10-08") +HALVES = [("2016-02-01", "2020-12-31"), ("2021-01-01", "2026-10-08")] +TOP, RISK, MAX_LEV, ATR_STOP = 20, 0.01, 4.0, 0.10 +COMMISSION, STRESS = 0.0035, 0.0005 + +PREREG = { + "question": "Does the 5-minute ORB on the top-20 stocks in play replicate on the PIT S&P 1500 with " + "5-minute bars, and does it hold after the paper's sample?", + "rules": "paper text: price>5, avg vol14>=1M, ATR14>0.5, relvol>=1, top 20 relvol, OR direction, " + "stop entry at OR high/low, stop 10% ATR, exit at close, risk 1%, notional cap 4x/20", + "deviations": ["S&P 1500 PIT universe", "5-min bars, same-bar stop counted as hit"], + "continue_if_all": [ + "O1 net Sharpe >= 1.5 over 2016-02..2023-12 (the paper's sample)", + "O2 net Sharpe >= 1.0 over 2024-01..2026-10 (after it)", + "O3 net Sharpe >= 1.0 over 2016-26 with + 5 bp per side slippage", + "O4 net Sharpe >= 1.0 in 2016-20 and in 2021-26", + ], + "bounds": "primary = conservative (a stop inside the entry bar counts as hit); optimistic = the stop is " + "checked only from the bar after the entry. Both are run with the same gates.", + "decision": {"CONTINUE": "conservative passes: widen to all US stocks with 1-min bars, then paper-trade", + "RESOLVE": "only the optimistic bound passes: fetch 1-min bars for the ambiguous name-days " + "and rerun (a new study)", + "STOP": "the optimistic bound fails too: drop ORB on stocks in play"}, +} + + +def day_matrix(df: pd.DataFrame) -> dict: + idx = df.index + slot = ((idx.hour * 60 + idx.minute - 570) // 5).to_numpy() + dates = idx.tz_localize(None).normalize() + days = pd.DatetimeIndex(sorted(set(dates))) + row = days.get_indexer(dates) + out = {"days": days} + for k in ("o", "h", "l", "c", "v"): + a = np.full((len(days), 78), np.nan) + a[row, slot] = df[k].to_numpy(dtype=float) + out[k] = a + return out + + +def trades(df: pd.DataFrame, optimistic: bool = False) -> pd.DataFrame: + """Per day: eligibility inputs and the trade's return per unit of notional.""" + M = day_matrix(df) + O, H, L, C, V = M["o"], M["h"], M["l"], M["c"], M["v"] + D = len(M["days"]) + has_or = np.isfinite(O[:, 0]) & np.isfinite(C[:, 0]) + last_c = pd.DataFrame(C).ffill(axis=1).iloc[:, -1].to_numpy() + d_open = O[:, 0] + d_high, d_low = np.nanmax(H, axis=1), np.nanmin(L, axis=1) + d_vol = np.nansum(V, axis=1) + prev_c = np.r_[np.nan, last_c[:-1]] + tr = np.fmax(d_high - d_low, np.fmax(np.abs(d_high - prev_c), np.abs(d_low - prev_c))) + atr = pd.Series(tr).rolling(14, min_periods=14).mean().shift(1).to_numpy() + avgvol = pd.Series(d_vol).rolling(14, min_periods=14).mean().shift(1).to_numpy() + orv = np.where(has_or, V[:, 0], np.nan) + relvol = orv / pd.Series(orv).rolling(14, min_periods=10).mean().shift(1).to_numpy() + direction = np.sign(C[:, 0] - O[:, 0]) + + ret = np.full(D, np.nan) + entry_px = np.full(D, np.nan) + j = np.arange(78)[None, :] + for sgn in (1, -1): + rows = np.where((direction == sgn) & has_or & np.isfinite(atr))[0] + if len(rows) == 0: + continue + lvl = H[rows, 0] if sgn == 1 else L[rows, 0] + hh, ll, oo = H[rows], L[rows], O[rows] + trig = (hh > lvl[:, None]) if sgn == 1 else (ll < lvl[:, None]) + trig[:, 0] = False + trig &= np.isfinite(hh) + any_t = trig.any(axis=1) + k = np.argmax(trig, axis=1) + ok = np.where(any_t)[0] + r_ = rows[ok] + kk = k[ok] + o_k = oo[ok, kk] + ent = np.where(np.isfinite(o_k), np.fmax(lvl[ok], o_k) if sgn == 1 else np.fmin(lvl[ok], o_k), lvl[ok]) + stop = ent - sgn * ATR_STOP * atr[r_] + hit = (ll[ok] <= stop[:, None]) if sgn == 1 else (hh[ok] >= stop[:, None]) + hit &= (j > kk[:, None]) if optimistic else (j >= kk[:, None]) + any_h = hit.any(axis=1) + h = np.argmax(hit, axis=1) + o_h = oo[ok, h] + gap_fill = np.where(np.isfinite(o_h), np.fmin(stop, o_h) if sgn == 1 else np.fmax(stop, o_h), stop) + stop_fill = np.where(h == kk, stop, gap_fill) # same bar as the entry: at the stop + exit_px = np.where(any_h, stop_fill, last_c[r_]) + ret[r_] = sgn * (exit_px / ent - 1) + entry_px[r_] = ent + return pd.DataFrame({"open": d_open, "atr": atr, "avgvol": avgvol, "relvol": relvol, "dir": direction, + "ret": ret, "entry": entry_px}, index=M["days"]) + + +def members() -> pd.DataFrame: + from research.lab.factor_scan import load + from research.lab.panel_mid import load_us_mid + a, _ = load("us") + b = load_us_mid("midsmall") + m = pd.concat([a.member, b.member], axis=1) + return m.T.groupby(level=0).any().T.loc["2015-12-01":] + + +def book(T: pd.DataFrame, extra: float) -> pd.DataFrame: + """T: rows (date, symbol) of the day's selected names. Daily return on equity.""" + T = T.copy() + risk_notional = RISK * T["entry"] / (ATR_STOP * T["atr"]) # notional / equity at 1% risk + T["w"] = np.minimum(risk_notional, MAX_LEV / TOP) + T["cost"] = 2 * (COMMISSION / T["entry"] + extra) + T["pnl"] = T["w"] * (T["ret"] - T["cost"]) + g = T.groupby(level="date") + return pd.DataFrame({"ret": g["pnl"].sum(), "n": g.size(), "gross_w": g["w"].sum()}) + + +def describe(r: pd.Series) -> dict: + eq = (1 + r).cumprod() + yrs = len(r) / 252 + return {"cagr": float(eq.iloc[-1] ** (1 / yrs) - 1), "vol": float(r.std() * np.sqrt(252)), + "sharpe": float(r.mean() / r.std() * np.sqrt(252)), "mdd": float((eq / eq.cummax() - 1).min())} + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + mem = members() + files = sorted(f for f in os.listdir(BARS) if f.endswith(".pkl")) + parts = [] + for i, f in enumerate(files, 1): + sym = f[:-4] + bars = pd.read_pickle(os.path.join(BARS, f)) + t = trades(bars) + t["ret_opt"] = trades(bars, optimistic=True)["ret"] + if sym in mem.columns: + t["member"] = mem[sym].reindex(t.index).fillna(False).to_numpy() + else: + t["member"] = False + t = t[(t["dir"] != 0) & t["member"] & (t["open"] > 5) & (t["avgvol"] >= 1e6) & (t["atr"] > 0.5) + & (t["relvol"] >= 1)] + t["symbol"] = sym + parts.append(t) + if i % 250 == 0: + print(f" {i}/{len(files)}", flush=True) + A = pd.concat(parts) + A.index.name = "date" + A = A.set_index("symbol", append=True) + print(f"eligible name-days {len(A)}, days {A.index.get_level_values(0).nunique()}") + sel = A.groupby(level="date", group_keys=False).apply(lambda g: g.nlargest(TOP, "relvol")) + traded = sel[sel["ret"].notna()] + res = {"universe_files": len(files), "eligible_name_days": int(len(A)), + "selected_name_days": int(len(sel)), "triggered": int(len(traded))} + days = pd.DatetimeIndex(sorted(set(A.index.get_level_values(0)))) + B = {} + opt = traded.assign(ret=traded["ret_opt"]) + for name, extra, T in (("paper_costs", 0.0, traded), ("stress_5bp", STRESS, traded), + ("optimistic_paper_costs", 0.0, opt), ("optimistic_stress_5bp", STRESS, opt)): + b = book(T, extra).reindex(days).fillna({"ret": 0.0, "n": 0, "gross_w": 0.0}) + B[name] = b + r = b["ret"].loc[SPAN[0]:SPAN[1]] + d = {"full": describe(r), "paper": describe(r.loc[PAPER[0]:PAPER[1]]), "post": describe(r.loc[POST[0]:]), + "halves": [describe(r.loc[a:z]) for a, z in HALVES], + "avg_positions": float(b["n"].loc[SPAN[0]:].mean()), + "avg_gross_leverage": float(b["gross_w"].loc[SPAN[0]:].mean())} + yr = (1 + r).groupby(r.index.year).prod() - 1 + d["yearly"] = {int(k): float(v) for k, v in yr.items()} + res[name] = d + print(f"{name:12s} full cagr {d['full']['cagr']*100:6.1f}% sh {d['full']['sharpe']:5.2f} mdd " + f"{d['full']['mdd']*100:6.1f}% | 2016-23 sh {d['paper']['sharpe']:5.2f} | 2024- sh " + f"{d['post']['sharpe']:5.2f} cagr {d['post']['cagr']*100:6.1f}% | halves " + f"{d['halves'][0]['sharpe']:5.2f} {d['halves'][1]['sharpe']:5.2f} | pos {d['avg_positions']:.1f} " + f"lev {d['avg_gross_leverage']:.2f}") + print(" yearly %:", {k: round(v * 100, 1) for k, v in d["yearly"].items()}) + tr = traded.loc[SPAN[0]:] + res["trade_stats"] = {"n": int(len(tr)), "win_rate": float((tr["ret"] > 0).mean()), + "mean_ret_bp": float(tr["ret"].mean() * 1e4), + "mean_R": float((tr["ret"] * tr["entry"] / (ATR_STOP * tr["atr"])).mean())} + print("trades:", res["trade_stats"]) + + def gates(p, s): + return {"O1": p["paper"]["sharpe"] >= 1.5, "O2": p["post"]["sharpe"] >= 1.0, + "O3": s["full"]["sharpe"] >= 1.0, "O4": all(h["sharpe"] >= 1.0 for h in p["halves"])} + checks = gates(res["paper_costs"], res["stress_5bp"]) + checks_opt = gates(res["optimistic_paper_costs"], res["optimistic_stress_5bp"]) + decision = ("CONTINUE" if all(checks.values()) else "RESOLVE" if all(checks_opt.values()) else "STOP") + res["verdict"] = {"checks": checks, "checks_optimistic": checks_opt, "decision": decision} + print(f"checks {checks} optimistic {checks_opt} -> {decision}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.concat(B, axis=1).to_pickle(os.path.join(OUT, "daily.pkl")) + traded.to_pickle(os.path.join(OUT, "trades.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"orb_stocks_in_play:{k}", **res[k]["full"]} for k in B], + {"study": "orb_stocks_in_play", "role": "preregistered replication"}) + + +if __name__ == "__main__": + main() diff --git a/research/lab/short_vol.py b/research/lab/short_vol.py new file mode 100644 index 0000000..f079645 --- /dev/null +++ b/research/lab/short_vol.py @@ -0,0 +1,156 @@ +""" +Quick check: is the volatility risk premium (short VIX futures, out when the +VIX term structure inverts) a stream with a Sharpe high enough to lever toward +50% a year, and does it survive 2008, 2018-02 and 2020-03? + + PYTHONPATH=. uv run python -m research.lab.short_vol [--no-proxy] + +Data (Yahoo): + ^SHORTVOL Cboe short VIX futures index from 2005-12: daily -1x short-term + VIX futures (daily corr with -VIXY 0.987); treated as an excess + return, so collateral earns the T-bill + ^VIX, ^VIX3M spot VIX and 3-month VIX (from 2006-07) +Executable as SVXY (-0.5x, 0.95%/yr) or short VX / VXM futures. + +Strategy (Cooper 2013 "Easy Volatility Investing", published 2013-11): + signal short vol for day t+1 when VIX/VIX3M at the close of t < 1 (contango) + size weight = min(1, 20% / EWMA-21 vol of ^SHORTVOL through t) + return rf + w x shortvol - 0.95%/yr x w - 10 bp x |dw| +Reported only: no filter; filter at 0.95; 30% vol target; unlevered -0.5x +(SVXY-like, no sizing). +Periods: 2006-07..2013-12 (before publication, includes 2008) and +2014-01..2026-09 (after). + +The rule is written to prereg.json before the run; the script refuses to +overwrite results.json. The project has not tested volatility selling before. + +Result (2026-10-08): STOP. C2 and C4 pass; C1 and C3 fail. + - vrp_ts_vt20, full span: 12.9% / Sharpe 0.61 / MaxDD -36.5%. + - Before publication (2006-13): Sharpe 1.05. + - After (2014-26): 7.8% / Sharpe 0.37. + - Worst day -15.7% (2007-02-27). Bad years: 2018 -24%, 2024 -10%, 2020 -8%. + - The term-structure filter holds MaxDD to -36% (no filter: -52%) but does not + lift the post-2014 Sharpe (0.33-0.37 in every variant). + - SVXY-like -0.5x held throughout: 2014-26 Sharpe 0.52, MaxDD -64%, worst day + -21% (2024-08-05). + - Monthly corr: SPY 0.58, TrendRider 0.33. It is equity beta with a short + crash option, not a diversifier. +Interpretation: the volatility premium shrank after 2013. What is left does +not pay for the tails it carries, at any leverage. +""" + +from __future__ import annotations + +import argparse +import json +import os + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS +from research.lab.levered_book import stats, tbill + +OUT = os.path.join(RESULTS, "short_vol") +PRE, POST = ("2006-08-01", "2013-12-31"), ("2014-01-01", "2026-09-24") +FEE, COST = 0.0095, 0.001 + +PREREG = { + "question": "Is term-structure-filtered short VIX futures a high-Sharpe stream that survives its crashes?", + "primary": "vrp_ts_vt20: short ^SHORTVOL when VIX/VIX3M < 1 at the prior close, sized to 20% vol (cap 1x)", + "continue_if_all": [ + "C1 2014-2026 (post-publication) excess Sharpe >= 1.0", + "C2 2006-2013 excess Sharpe >= 0.5", + "C3 full-span MaxDD >= -35% and worst single day >= -10%", + "C4 monthly corr with TrendRider V10 <= 0.5", + ], + "decision": {"CONTINUE": "preregister a VRP sleeve in the book (instrument SVXY / VXM, leverage, tail limits)", + "STOP": "drop short volatility"}, +} + + +def strategy(sv: pd.Series, ratio: pd.Series, rf: pd.Series, thr: float | None, target: float | None, + fixed: float | None = None) -> tuple[pd.Series, pd.Series]: + vol = sv.ewm(span=21, min_periods=21).std() * np.sqrt(252) + if fixed is not None: + w = pd.Series(fixed, index=sv.index) + else: + w = (target / vol).clip(upper=1.0) + if thr is not None: + w = w.where(ratio < thr, 0.0) + w = w.shift(1).fillna(0.0) + ret = rf / 252 + w * sv.fillna(0.0) - w * FEE / 252 - w.diff().abs().fillna(0.0) * COST + return ret, w + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--no-proxy", action="store_true") + if not ap.parse_args().no_proxy: + from research.lab.fetch_us import _proxy_env + _proxy_env() + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + from research.lab.fetch_long_history import fetch_yahoo + c, _ = fetch_yahoo(["^SHORTVOL", "^VIX", "^VIX3M"], start="2005-01-01") + c = c.loc[:POST[1]].dropna(subset=["^SHORTVOL"]) + sv = c["^SHORTVOL"].pct_change(fill_method=None) + ratio = c["^VIX"] / c["^VIX3M"] + rf = tbill(c.index).fillna(0.0) + variants = { + "vrp_ts_vt20": (1.0, 0.20, None), + "vrp_ts95_vt20": (0.95, 0.20, None), + "vrp_ts_vt30": (1.0, 0.30, None), + "vrp_nofilter_vt20": (None, 0.20, None), + "svxy_like_half_nofilter": (None, None, 0.5), + } + R, res = {}, {"variants": {}} + for k, (thr, tgt, fixed) in variants.items(): + r, w = strategy(sv, ratio, rf, thr, tgt, fixed) + r = r.loc[PRE[0]:POST[1]] + R[k] = r + row = {} + for name, (a, b) in {"full": (PRE[0], POST[1]), "pre": PRE, "post": POST}.items(): + s = stats(r.loc[a:b], rf.loc[a:b]) + row[name] = {x: float(s[x]) for x in ("cagr", "sharpe", "vol", "mdd", "worst_yr")} + row["worst_day"] = float(r.min()) + row["worst_days"] = {str(d.date()): float(v) for d, v in r.nsmallest(5).items()} + row["time_in"] = float((w.loc[PRE[0]:] > 0).mean()) + row["avg_w"] = float(w.loc[PRE[0]:].mean()) + res["variants"][k] = row + f, p, q = row["full"], row["pre"], row["post"] + print(f"{k:24s} full {f['cagr']*100:6.1f}% xsh {f['sharpe']:5.2f} mdd {f['mdd']*100:6.1f}% | pre xsh " + f"{p['sharpe']:5.2f} mdd {p['mdd']*100:6.1f}% | post xsh {q['sharpe']:5.2f} cagr {q['cagr']*100:5.1f}% " + f"mdd {q['mdd']*100:6.1f}% | worst day {row['worst_day']*100:5.1f}% in {row['time_in']:.2f} " + f"avg w {row['avg_w']:.2f}") + print(" worst days:", {d: round(v * 100, 1) for d, v in row["worst_days"].items()}) + yr = (1 + R["vrp_ts_vt20"]).groupby(R["vrp_ts_vt20"].index.year).prod() - 1 + print("vrp_ts_vt20 yearly %:", (yr * 100).round(1).to_dict()) + res["yearly_primary"] = {int(k): float(v) for k, v in yr.items()} + S = pd.read_pickle(os.path.join(RESULTS, "stream_portfolio", "streams.pkl")) + m = lambda x: (1 + x.dropna()).resample("ME").prod() - 1 + res["corr_trend"] = float(m(R["vrp_ts_vt20"]).corr(m(S["trend"]))) + res["corr_spy"] = float(m(R["vrp_ts_vt20"]).corr(m(S["spy"]))) + print(f"monthly corr: TrendRider {res['corr_trend']:.2f}, SPY {res['corr_spy']:.2f}") + + p = res["variants"]["vrp_ts_vt20"] + checks = {"C1": p["post"]["sharpe"] >= 1.0, "C2": p["pre"]["sharpe"] >= 0.5, + "C3": p["full"]["mdd"] >= -0.35 and p["worst_day"] >= -0.10, "C4": res["corr_trend"] <= 0.5} + res["verdict"] = {"checks": checks, "decision": "CONTINUE" if all(checks.values()) else "STOP"} + print(f"checks {checks} -> {res['verdict']['decision']}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.DataFrame(R).to_pickle(os.path.join(OUT, "returns.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"short_vol:{k}", **v["full"]} for k, v in res["variants"].items()], + {"study": "short_vol", "role": "preregistered quick check"}) + + +if __name__ == "__main__": + main() diff --git a/research/lab/small_cap_reversal.py b/research/lab/small_cap_reversal.py new file mode 100644 index 0000000..aa037a0 --- /dev/null +++ b/research/lab/small_cap_reversal.py @@ -0,0 +1,212 @@ +""" +Quick check: is short-term reversal (liquidity provision) in S&P 400/600 +names a market-neutral stream strong enough to lever, now that a $1M account +can trade where large funds cannot? + + PYTHONPATH=. uv run python -m research.lab.small_cap_reversal + +Panels: the PIT S&P 400/600 union (research/lab/panel_mid.py, Tiingo +extended; member coverage 77% of the 1000 names in 2017, 99% in 2026) and the +PIT S&P 500 (factor_scan.load("us")) as a large-cap consistency check. + +Strategy (Lehmann 1990; Nagel 2012), close prices >= $5 only: + signal 5-day return minus the cross-sectional mean, from closes through t + basket the lowest decile (losers), equal weight + holding enter at the open of t+1, exit at the open of t+6 (five + overlapping tranches, each 1/5 of the book; at least 5 sessions held) + hedge short the equal-weight member universe (beta 1), which an + equal-weight index future or ETF approximates; the hedge's own + costs are ignored here + returns open-to-open, so book_d = sum over L=2..6 of 1/5 x mean(r_d over basket_{d-L}) + costs per name and side: Abdi-Ranaldo (2017) half-spread from daily + high/low/close (21-day mean) + 2 bp, charged on the change in the + aggregate weights; a flat 20 bp/side variant is reported too + +The rule is written to prereg.json before the run; the script refuses to +overwrite results.json. 2017-26 of the mid/small panel has not been used for a +reversal test before; the S&P 500 reversal sleeves of short_horizon.py were +long-only and monthly/weekly, so this is not a repeat but not untouched data. + +Result (2026-10-08): STOP. C1, C2 and C3 all fail. + - midsmall rev5 hedged: gross Sharpe 0.58 (about 7%/yr at 12% vol). Net is + -22%/yr at the Abdi-Ranaldo cost (median 17 bp a side) and -10%/yr at a + flat 20 bp. Every year 2017-2026 is negative. + - Turnover is 0.34 of the book a day (~86x a year), so breakeven is about + 8 bp a side, under what MOO orders in S&P 400/600 names cost. + - S&P 500 2006-26: gross 0.61; net -1.34 (-0.83 at 20 bp). + - Quiet (low abnormal volume) losers are weaker, not stronger (gross 0.45). + - Bug in the reported-only long_short row: net(losers) - net(winners) + adds the winners' costs back. That row (0.46 / 0.57) is near gross, + not net. +Interpretation: daily-frequency liquidity provision earns less than the +spread a non-market-maker pays. The edge sits intraday, with market makers. +""" + +from __future__ import annotations + +import json +import os + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS, load +from research.lab.levered_book import stats, tbill +from research.lab.panel_mid import load_us_mid + +OUT = os.path.join(RESULTS, "small_cap_reversal") +SPAN_MID = ("2017-01-03", "2026-09-24") +HALVES_MID = [("2017-01-03", "2021-12-31"), ("2022-01-03", "2026-09-24")] +SPAN_LARGE = ("2006-10-02", "2026-09-24") +HOLD, LOOK, MIN_PRICE, FEE = 5, 5, 5.0, 0.0002 + +PREREG = { + "question": "Is 5-day reversal in S&P 400/600 names, hedged with the EW universe, a stream to lever?", + "primary": "midsmall rev5 losers decile, hold 5, EW-universe hedge, Abdi-Ranaldo costs, 2017-01..2026-09", + "continue_if_all": [ + "C1 primary net Sharpe >= 1.0 over 2017-26", + "C2 primary net Sharpe >= 0.5 in 2017-21 and in 2022-26", + "C3 S&P 500 version net Sharpe >= 0.3 over 2006-26 (the effect exists beyond one panel)", + ], + "reported_not_decisive": ["gross Sharpe", "flat 20 bp/side costs", "long-short (losers - winners)", + "low-abnormal-volume losers (Nagel/Avramov: no-news moves revert more)", + "turnover", "monthly corr with the three-sleeve book"], + "decision": {"CONTINUE": "full study: hedge instrument, capacity at $1M, MOO execution, leverage", + "STOP": "drop short-horizon liquidity provision at daily frequency"}, + "sharpe": "hedged stream: mean/std of daily net returns, annualized (no T-bill: it is self-financing " + "on top of cash)", +} + + +def half_spread(p) -> pd.DataFrame: + """Abdi-Ranaldo (2017) close-high-low half-spread estimate, 21-day mean, floored at 0.""" + c = np.log(p.close) + eta = (np.log(p.high) + np.log(p.low)) / 2 + s2 = 4 * (c - eta) * (c - eta.shift(-1)) + s2 = s2.shift(1) # uses eta through t only after the shift + s = np.sqrt(s2.rolling(21, min_periods=10).mean().clip(lower=0)) + return (s / 2).clip(upper=0.02) + + +def tranche_weights(score: pd.DataFrame, pick: pd.DataFrame) -> pd.DataFrame: + w = pick.astype(float) + return w.div(w.sum(axis=1).replace(0, np.nan), axis=0).fillna(0.0) + + +def book(p, basket: pd.DataFrame, universe: pd.DataFrame, cost_side: pd.DataFrame | float, + hedge: bool = True) -> pd.DataFrame: + """Daily open-to-open returns of the overlapping-tranche book, gross and net.""" + ro = (p.open / p.open.shift(1) - 1).where(lambda x: x.abs() < 1.0) + w = tranche_weights(None, basket) + H = sum(w.shift(L) for L in range(2, 2 + HOLD)) / HOLD + gross = (H * ro.fillna(0.0)).sum(axis=1) + if hedge: + u = universe.shift(2) # hedge sized to the same information + gross = gross - ro.where(u).mean(axis=1).fillna(0.0) * (H.sum(axis=1) > 0) + dH = (H - H.shift(1)).abs() + cost = (dH * cost_side).sum(axis=1) if isinstance(cost_side, pd.DataFrame) else dH.sum(axis=1) * cost_side + return pd.DataFrame({"gross": gross, "net": gross - cost, "turnover": dH.sum(axis=1)}) + + +def signals(p): + c = p.close + ok = p.member & (c >= MIN_PRICE) & c.notna() & p.open.notna() + r5 = (c / c.shift(LOOK) - 1).where(ok) + adj = r5.sub(r5.mean(axis=1), axis=0) + rk = adj.rank(axis=1, pct=True) + losers = rk <= 0.10 + winners = rk > 0.90 + dv = (c * p.volume) + abn = (dv.rolling(LOOK).mean() / dv.rolling(63, min_periods=40).mean().shift(LOOK)).where(ok) + quiet = losers & (abn.rank(axis=1, pct=True) <= 0.5) + return ok, losers, winners, quiet + + +def sharpe(r: pd.Series) -> float: + r = r.dropna() + return float(r.mean() / r.std() * np.sqrt(252)) if r.std() > 0 else float("nan") + + +def describe(r: pd.Series, span, halves) -> dict: + r = r.loc[span[0]:span[1]] + eq = (1 + r).cumprod() + return {"ann_ret": float(r.mean() * 252), "vol": float(r.std() * np.sqrt(252)), "sharpe": sharpe(r), + "mdd": float((eq / eq.cummax() - 1).min()), + "halves": [sharpe(r.loc[a:b]) for a, b in halves]} + + +def run_panel(p, span, halves, tag: str) -> tuple[dict, pd.DataFrame]: + ok, losers, winners, quiet = signals(p) + hs = half_spread(p) + FEE + hs = hs.fillna(hs.stack().median()) + out, R = {}, {} + variants = { + "rev5_hedged": (losers, hs, True), + "rev5_hedged_flat20": (losers, 0.002, True), + "rev5_quiet_hedged": (quiet, hs, True), + "rev5_unhedged": (losers, hs, False), + } + for k, (bk, cs, hedge) in variants.items(): + b = book(p, bk, ok, cs, hedge) + R[f"{k}_net"], R[f"{k}_gross"] = b["net"], b["gross"] + d = describe(b["net"], span, halves) + d["gross_sharpe"] = sharpe(b["gross"].loc[span[0]:span[1]]) + d["turnover_per_day"] = float(b["turnover"].loc[span[0]:span[1]].mean()) + out[k] = d + lw = book(p, losers, ok, hs, False)["net"] - book(p, winners, ok, hs, False)["net"] + R["long_short_net"] = lw + out["long_short"] = describe(lw, span, halves) + med = hs.where(ok).loc[span[0]:span[1]].stack().median() + out["median_half_spread_plus_fee_bp"] = float(med * 1e4) + for k, d in out.items(): + if isinstance(d, dict): + print(f"[{tag}] {k:22s} ann {d['ann_ret']*100:6.1f}% vol {d['vol']*100:5.1f}% sh {d['sharpe']:5.2f} " + f"mdd {d['mdd']*100:6.1f}% halves {d['halves'][0]:5.2f} {d['halves'][1]:5.2f}" + + (f" gross sh {d['gross_sharpe']:5.2f} turn/day {d['turnover_per_day']:.2f}" + if "gross_sharpe" in d else "")) + print(f"[{tag}] median cost per side {med*1e4:.1f} bp") + return out, pd.DataFrame(R) + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + res = {} + mid = load_us_mid("midsmall") + res["midsmall"], Rm = run_panel(mid, SPAN_MID, HALVES_MID, "midsmall") + large, _ = load("us") + res["sp500"], Rl = run_panel(large, SPAN_LARGE, [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")], + "sp500") + yr = (1 + Rm["rev5_hedged_net"].loc[SPAN_MID[0]:SPAN_MID[1]]).groupby( + Rm.loc[SPAN_MID[0]:SPAN_MID[1]].index.year).prod() - 1 + print("midsmall rev5_hedged net yearly %:", (yr * 100).round(1).to_dict()) + res["yearly_mid_primary"] = {int(k): float(v) for k, v in yr.items()} + try: + S = pd.read_pickle(os.path.join(RESULTS, "managed_futures_book", "books.pkl"))["three_sleeve_v7"] + m = lambda x: (1 + x.dropna()).resample("ME").prod() - 1 + res["corr_three_sleeve"] = float(m(Rm["rev5_hedged_net"].loc[SPAN_MID[0]:]).corr(m(S))) + print(f"monthly corr with three_sleeve_v7: {res['corr_three_sleeve']:.2f}") + except FileNotFoundError: + pass + + pm, pl = res["midsmall"]["rev5_hedged"], res["sp500"]["rev5_hedged"] + checks = {"C1": pm["sharpe"] >= 1.0, "C2": all(x >= 0.5 for x in pm["halves"]), "C3": pl["sharpe"] >= 0.3} + res["verdict"] = {"checks": checks, "decision": "CONTINUE" if all(checks.values()) else "STOP"} + print(f"checks {checks} -> {res['verdict']['decision']}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.concat({"midsmall": Rm, "sp500": Rl}, axis=1).to_pickle(os.path.join(OUT, "returns.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"small_cap_reversal:{panel}:{k}", **v} for panel in ("midsmall", "sp500") + for k, v in res[panel].items() if isinstance(v, dict)], + {"study": "small_cap_reversal", "role": "preregistered quick check"}) + + +if __name__ == "__main__": + main() diff --git a/research/lab/turn_of_month.py b/research/lab/turn_of_month.py new file mode 100644 index 0000000..9ef4bbe --- /dev/null +++ b/research/lab/turn_of_month.py @@ -0,0 +1,155 @@ +""" +Quick check: does the turn-of-the-month effect (Ariel 1987; Lakonishok & +Smidt 1988; McConnell & Xu 2008) still hold after publication, and is it +strong enough to carry leverage (3x the S&P 500 for ~4 days a month, T-bills +otherwise)? + + PYTHONPATH=. uv run python -m research.lab.turn_of_month + +Data: S&P 500 price index (^GSPC) from 1928 (no dividends; ~2%/yr spread +evenly, so it barely moves a 4-day vs 17-day comparison), the 3-month T-bill +from FRED DTB3 (from 1954; 0 before). + +Window (fixed in the literature, not tuned): the last trading day of a month +and the first three of the next, i.e. bought at the close of day -2 and sold +at the close of day +3. The calendar is known in advance, so close fills +are executable. + tom_1x S&P 500 in the window, T-bills otherwise + tom_3x 3x daily S&P 500 in the window: 3 r - 2 rf - 0.9%/yr fee (a UPRO-like + fund), T-bills otherwise +Periods: 1928-1987 (before publication), 1988-2007 (after), 2008-2026 (after +McConnell & Xu, the most recent). +Reported only: the pre-holiday day (Ariel 1990), inferred from a weekday gap. + +The rule is written to prereg.json before the run; the script refuses to +overwrite results.json. The project has not tested calendar effects before. + +Result (2026-10-08): STOP. C1, C2 and C3 all fail. + - In-window minus out-of-window excess return: + - 1928-87: 15.8 bp a day (t 6.3); the window earned 2.1x all the excess. + - 1988-2007: 9.2 bp (t 2.6). + - 2008-26: 1.2 bp (t 0.27). + - 1988-2026 pooled: t 1.86. + - t by decade: 1950s 5.8, 1960s 3.5, 1990s 1.4, 2000s 0.7, 2010s 0.2, 2020s 0.5. + - tom_3x 2008-26: 5.0%/yr, Sharpe 0.27, MaxDD -47%. SPX buy-and-hold: 9.3% / 0.48. + - Pre-holiday 2008-26: +16.6 bp (t 1.9), reported only, about 9 days a year. +Interpretation: a textbook post-publication decay. Little of the premium +is left to lever. +""" + +from __future__ import annotations + +import json +import os +import pickle + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS +from research.lab.levered_book import stats, tbill + +OUT = os.path.join(RESULTS, "turn_of_month") +PERIODS = {"1928-1987": ("1928-01-01", "1987-12-31"), "1988-2007": ("1988-01-01", "2007-12-31"), + "2008-2026": ("2008-01-01", "2026-09-24")} +POST = ("1988-01-01", "2026-09-24") +FEE3 = 0.009 + +PREREG = { + "question": "Does the turn-of-month premium survive publication and carry 3x leverage?", + "window": "trading days -1, +1, +2, +3 around each month end (bought at the close of day -2)", + "continue_if_all": [ + "C1 1988-2026: mean daily excess return in-window minus out-of-window, Welch t >= 2.0", + "C2 2008-2026: the same difference > 0 with t >= 1.5", + "C3 2008-2026: tom_3x excess Sharpe >= 0.5", + ], + "decision": {"CONTINUE": "preregister a TOM leverage overlay on the book (lever only in the window)", + "STOP": "drop calendar effects"}, +} + + +def welch(a: pd.Series, b: pd.Series) -> tuple[float, float]: + d = a.mean() - b.mean() + se = np.sqrt(a.var() / len(a) + b.var() / len(b)) + return float(d), float(d / se) + + +def tom_mask(idx: pd.DatetimeIndex) -> pd.Series: + """True on days -1, +1, +2, +3 of each month (return days, close-to-close).""" + m = idx.to_period("M") + s = pd.Series(m, index=idx) + pos = s.groupby(s).cumcount() + 1 # +1, +2, ... from month start + neg = s.groupby(s).cumcount(ascending=False) + 1 # 1 on the last day + return pd.Series((pos <= 3).to_numpy() | (neg == 1).to_numpy(), index=idx) + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + with open(os.path.join("data", "research", "long_history.pkl"), "rb") as fh: + px = pickle.load(fh)["close"]["^GSPC"].dropna() + px = px.loc[:"2026-09-24"] + r = px.pct_change().dropna() + idx = r.index + rf = tbill(idx).fillna(0.0) + ex = r - rf / 252 + tom = tom_mask(idx) + gap = pd.Series(idx, index=idx).shift(-1) - pd.Series(idx, index=idx) + pre_hol = (gap > pd.Timedelta(days=1)) & (pd.Series(idx.dayofweek, index=idx) < 4) # next day skipped mid-week + pre_hol |= (gap > pd.Timedelta(days=3)) + + tom_1x = rf / 252 + ex.where(tom, 0.0) + tom_3x = rf / 252 + (3 * ex - FEE3 / 252).where(tom, 0.0) + spx_3x = rf / 252 + 3 * ex - FEE3 / 252 + res = {"share_in_window": float(tom.mean()), "periods": {}} + print(f"share of days in window {tom.mean():.2%}") + for name, (a, b) in {**PERIODS, "1988-2026": POST}.items(): + e, t = ex.loc[a:b], tom.loc[a:b] + d, tt = welch(e[t], e[~t]) + dh, th = welch(e[pre_hol.loc[a:b] & ~t], e[~pre_hol.loc[a:b] & ~t]) + row = {"in_bp": float(e[t].mean() * 1e4), "out_bp": float(e[~t].mean() * 1e4), "diff_bp": d * 1e4, + "t": tt, "share_of_total_excess": float(e[t].sum() / e.sum()) if e.sum() > 0 else float("nan"), + "pre_holiday_diff_bp": dh * 1e4, "pre_holiday_t": th} + for k, s in (("tom_1x", tom_1x), ("tom_3x", tom_3x), ("spx_1x", r), ("spx_3x", spx_3x)): + st = stats(s.loc[a:b], rf.loc[a:b]) + row[k] = {"cagr": float(st["cagr"]), "sharpe": float(st["sharpe"]), "mdd": float(st["mdd"]), + "worst_yr": float(st["worst_yr"])} + res["periods"][name] = row + print(f"{name}: in {row['in_bp']:5.1f} bp/d out {row['out_bp']:5.1f} diff {row['diff_bp']:5.1f} t {tt:4.2f} " + f"| share of excess {row['share_of_total_excess']:.2f} | pre-hol diff {row['pre_holiday_diff_bp']:5.1f} t {th:4.2f}") + for k in ("tom_1x", "tom_3x", "spx_1x", "spx_3x"): + s = row[k] + print(f" {k:7s} cagr {s['cagr']*100:5.1f}% xsh {s['sharpe']:5.2f} mdd {s['mdd']*100:6.1f}% " + f"worst yr {s['worst_yr']*100:6.1f}%") + by_dec = ex.groupby(idx.year // 10 * 10).apply(lambda e: welch(e[tom.loc[e.index]], e[~tom.loc[e.index]])[1]) + print("t by decade:", by_dec.round(2).to_dict()) + res["t_by_decade"] = {int(k): float(v) for k, v in by_dec.items()} + try: + S = pd.read_pickle(os.path.join(RESULTS, "stream_portfolio", "streams.pkl")) + m = lambda x: (1 + x.dropna()).resample("ME").prod() - 1 + res["corr_trend"] = float(m(tom_3x.loc["2006-10":]).corr(m(S["trend"]))) + print(f"monthly corr tom_3x vs TrendRider V10: {res['corr_trend']:.2f}") + except FileNotFoundError: + pass + + p1, p2 = res["periods"]["1988-2026"], res["periods"]["2008-2026"] + checks = {"C1": p1["t"] >= 2.0, "C2": p2["diff_bp"] > 0 and p2["t"] >= 1.5, "C3": p2["tom_3x"]["sharpe"] >= 0.5} + res["verdict"] = {"checks": checks, "decision": "CONTINUE" if all(checks.values()) else "STOP"} + print(f"checks {checks} -> {res['verdict']['decision']}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.DataFrame({"tom_1x": tom_1x, "tom_3x": tom_3x, "in_window": tom}).to_pickle(os.path.join(OUT, "returns.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"turn_of_month:{p}:{k}", **res["periods"][p][k]} for p in res["periods"] + for k in ("tom_1x", "tom_3x")], + {"study": "turn_of_month", "role": "preregistered quick check"}) + + +if __name__ == "__main__": + main()