Compare commits
4 Commits
0e1e0a202c
...
05c2ac4c78
| Author | SHA1 | Date | |
|---|---|---|---|
| 05c2ac4c78 | |||
| 9287a92461 | |||
| 81f4e3b32c | |||
| ed0ed3f4b2 |
13
CLAUDE.md
13
CLAUDE.md
@@ -106,6 +106,13 @@ PYTHONPATH=. uv run python -m research.lab.overnight_book # three-slee
|
||||
PYTHONPATH=. uv run python -m research.lab.overnight_book_mf [--no-proxy] # the overnight books + a DBMF/CTA sleeve, levered to 50%/yr (preregistered; KEEP 2026-10-09: levered MaxDD unchanged, no CTA sleeve)
|
||||
PYTHONPATH=. uv run python -m research.lab.daytime_reversal # buy the S&P 500's biggest overnight losers at 09:35, sell at the close (preregistered; STOP 2026-10-09: gross 1-6 bp/day, net Sharpe ~0)
|
||||
PYTHONPATH=. uv run python -m research.lab.overnight_midsmall # the overnight-only momentum book on S&P 400/600 (preregistered; STOP 2026-10-09: 0.93 net at 3 bp/side, corr 0.83 with the S&P 500 book)
|
||||
PYTHONPATH=. uv run --with numba python -m research.lab.dip_buying # buy-the-dip: SPY/QQQ RSI(2), trend-filtered stock RSI(2), drawdown tables (preregistered; STOP 2026-10-12: SPY 46 bp/trade t 3.4 but 14% of days, 3x Sharpe 0.57 = SPY; stocks breakeven 10 bp/side)
|
||||
PYTHONPATH=. uv run python -m research.lab.dip_leverage_overlay # +1x SPY only inside SPY RSI(2) dip windows, on the overnight_book books (preregistered; STOP 2026-10-12: Sharpe 0.98->0.99 / 1.38->1.33, placebo rank 77-88%; the books already earn the bounce at ~2x SPY beta)
|
||||
PYTHONPATH=. uv run --with numba python -m research.lab.factor_census [--part construction] # every lab factor (price, EDGAR, insider, Lazy Prices, EAR, SVR, FTD) measured one way on the current PIT panel: IC, quintile spread, top-20 alpha vs EW (2026-10-12: 1 A low_svr21, 2 B accruals/rev_5, 69 N; no |t|>=3)
|
||||
PYTHONPATH=. uv run --with numba python -m research.lab.svr_combo # low_svr21 blended into overnight_recovery_top10 / as a fourth book sleeve, 2018-26 (preregistered; STOP 2026-10-12: blend Sharpe 0.81->0.87 but MaxDD -35%->-41%; book 1.49->1.42)
|
||||
PYTHONPATH=. uv run python -m research.lab.fetch_options snapshot|status # daily S&P 500 option chains after the close: indicative quotes (forward-only from 2026-10-12) + OPRA daily bars + print-vs-quote quality sample; data/options/us/; timer quant-trading-options
|
||||
PYTHONPATH=. uv run python -m research.lab.fetch_options backfill # OPRA daily option bars since 2024-02 for every S&P 500 member since then (resumable, ~2-3 h; pauses 16:10-16:55 NY); then builds volume_panel.csv.gz
|
||||
PYTHONPATH=. uv run python -m research.lab.fetch_options volume-panel # date x underlying option volume (all 0-100 DTE strikes and the 5-100 DTE / 0.7-1.3 band), os_volume, pc_volume
|
||||
uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs)
|
||||
uv run python -m research.lab.fetch_us_fundamentals --universe small # the same for S&P 600 (IJR)
|
||||
PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] [--universe small] # S&P 400 sleeves vs IJH; S&P 600 replication vs IJR (all REJECT 2026-10-08)
|
||||
@@ -207,7 +214,7 @@ The comparison writes its fixed budget and acceptance rule before running and re
|
||||
|
||||
## Server Deployment
|
||||
|
||||
Four `systemd --user` services, defined in `deploy/systemd/`, run from `/home/gahow/projects/quant-trading`:
|
||||
Four `systemd --user` services and one timer, defined in `deploy/systemd/`, run from `/home/gahow/projects/quant-trading`:
|
||||
|
||||
| Service | Script | Role |
|
||||
|---|---|---|
|
||||
@@ -215,11 +222,15 @@ Four `systemd --user` services, defined in `deploy/systemd/`, run from `/home/ga
|
||||
| `quant-trading-api` | `deploy/start-web-api.sh` | Web API on `127.0.0.1:8787` |
|
||||
| `quant-trading-vite` | `deploy/start-web-vite.sh` | Vite on `127.0.0.1:5174`, behind nginx on `:5443` |
|
||||
| `quant-trading-overnight-paper` | `deploy/start-overnight-paper.sh` | `overnight_paper.py daemon`: overnight-only momentum on the Alpaca paper account (from 2026-10-12) |
|
||||
| `quant-trading-options` (timer, 16:25 New York Mon-Fri) | `deploy/start-options-snapshot.sh` | `research.lab.fetch_options snapshot`: S&P 500 option chains (indicative quotes), OPRA daily bars, per-name IV/skew/spread/volume summary and a quote-quality sample into `data/options/us/` (~15 MB/day); `status` prints the indicative-vs-OPRA verdict after 20 sessions |
|
||||
|
||||
```bash
|
||||
cp deploy/systemd/*.service ~/.config/systemd/user/
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now quant-trading-monitor quant-trading-api quant-trading-vite
|
||||
systemctl --user enable --now quant-trading-options.timer
|
||||
# one-off, resumable: OPRA option-bar history since 2024-02 (rerun the same command if it stops)
|
||||
systemd-run --user --unit=quant-trading-options-backfill --working-directory=$PWD --setenv=PYTHONPATH=. uv run python -m research.lab.fetch_options backfill
|
||||
journalctl --user -u quant-trading-monitor -f
|
||||
```
|
||||
|
||||
|
||||
14
deploy/start-options-snapshot.sh
Executable file
14
deploy/start-options-snapshot.sh
Executable file
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Daily S&P 500 option-chain snapshot after the US close (research/lab/fetch_options.py).
|
||||
# Started by quant-trading-options.timer (16:25 America/New_York, Mon-Fri).
|
||||
# Keys are read from the repo-root .env by the script itself; Alpaca is reached directly (no proxy).
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
export PATH="/home/gahow/.local/bin:$PATH"
|
||||
export PYTHONPATH=.
|
||||
|
||||
exec uv run python -m research.lab.fetch_options snapshot
|
||||
10
deploy/systemd/quant-trading-options.service
Normal file
10
deploy/systemd/quant-trading-options.service
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily S&P 500 option-chain snapshot (Alpaca indicative feed)
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/gahow/projects/quant-trading
|
||||
ExecStart=/home/gahow/projects/quant-trading/deploy/start-options-snapshot.sh
|
||||
TimeoutStartSec=3600
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
9
deploy/systemd/quant-trading-options.timer
Normal file
9
deploy/systemd/quant-trading-options.timer
Normal file
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Option-chain snapshot after the US close
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Mon..Fri 16:25 America/New_York
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
493
research/lab/dip_buying.py
Normal file
493
research/lab/dip_buying.py
Normal file
@@ -0,0 +1,493 @@
|
||||
"""
|
||||
Quick check: does buying dips pay? The repo's reversal tests so far were
|
||||
cross-sectional (rev_5/rev_21/residual/sector reversal top 20, the hedged
|
||||
5-day S&P 400/600 reversal, the 09:35 gap-down buy). This one tests the two
|
||||
dip-buying rules the project has not tried, plus descriptive tables of what
|
||||
follows a drawdown.
|
||||
|
||||
PYTHONPATH=. uv run --with numba python -m research.lab.dip_buying
|
||||
|
||||
A. Index dip (Connors & Alvarez 2008, "RSI(2)"), SPY (gated) and QQQ (reported):
|
||||
enter close > 200-day mean and RSI(2) < 10 (Wilder smoothing)
|
||||
exit the first close above the 5-day mean
|
||||
fills next_open (gated): act at the open after the signal close;
|
||||
close (reported): act at the signal close, i.e. an MOC order sent
|
||||
with a 15:40 estimate of the signal
|
||||
otherwise T-bills (FRED DTB3)
|
||||
costs 1 bp a side (SPY/QQQ/UPRO spreads); 3 bp stress reported
|
||||
3x in-trade return 3 r - 2 rf - 0.9%/yr (a UPRO-like fund)
|
||||
pre-publication check (reported): ^GSPC 1962-1993, close fills, no costs
|
||||
|
||||
B. Stock dip with a trend filter, PIT S&P 500, $5+:
|
||||
enter close > 200-day mean and RSI(2) < 5; the lowest RSI(2) first,
|
||||
10 slots of 10% of equity
|
||||
exit the first close above the 5-day mean, 10 sessions held, or the
|
||||
name leaves the index; next_open fills (gated) / close (reported)
|
||||
costs 10 bp a side (the lab convention); breakeven reported
|
||||
also the repo's MeanReversionStrategy rule on the lab engine (12-month
|
||||
return > 0, the 20 biggest 1-month losers, monthly), reported only
|
||||
2006-2013 of this panel is survivor-biased (CLAUDE.md), which flatters a
|
||||
dip buyer most of all: the gate uses 2014-2026.
|
||||
|
||||
C. Descriptive, no gate: forward returns by drawdown from the 252-day high,
|
||||
for ^GSPC (1928-2026) and ^NDX (1986-2026), and for PIT S&P 500 stocks
|
||||
(excess over the member mean, entry at the next open).
|
||||
|
||||
The rule is written to prereg.json before the run; the script refuses to
|
||||
overwrite results.json.
|
||||
|
||||
Result (2026-10-12): STOP. C1 and C2 pass; C3, C4 and C5 fail.
|
||||
- Index RSI(2) is real but small. It is invested 14% of days, about 8 trades a year.
|
||||
SPY next_open trade bp t win 1x cagr/xSh 3x cagr/xSh/MaxDD SPY cagr/xSh
|
||||
1994-2007 73 4.7 78% 9.6% / 0.81 20.3% / 0.82 / -23% 10.4% / 0.43
|
||||
2008-2026 46 3.4 72% 4.9% / 0.56 10.9% / 0.57 / -41% 11.3% / 0.57
|
||||
2017-2026 39 2.4 74% 5.6% / 0.50 10.9% / 0.51 / -33% 15.3% / 0.74
|
||||
Close fills: 2008-26 53 bp (t 4.1), 3x 13.2% / 0.67. QQQ is similar
|
||||
(46 bp, t 3.4; 3x 0.49 vs QQQ 0.72). ^GSPC 1962-1993: 3.7 bp, t 0.36,
|
||||
so the effect is absent before the 1990s.
|
||||
- Reported only: always 1x, switched to 3x inside the dip trades. SPY
|
||||
2008-26: 17.5% / 0.69 / -52%, vs 11.3% / 0.57 / -52% for SPY. 2017-26:
|
||||
21.0% / 0.78 vs 15.3% / 0.74. QQQ 2008-26: 22.5% / 0.75 vs 16.3% / 0.72.
|
||||
- Stock RSI(2), 2014-26, next_open: -0.2%/yr, xSh -0.07. There were 5730
|
||||
trades with a 60% win rate and +0.8 bp net each; breakeven is 10.4 bp a
|
||||
side. With close fills: 2.1% / 0.09, breakeven 13 bp. 2006-13 with close
|
||||
fills (survivor-biased): 15.7% / 0.58. Adding it as a fourth sleeve
|
||||
lowers the book from 1.14 to 0.95. MeanReversionStrategy (monthly):
|
||||
2014-26 7.8% / 0.38, 2006-13 -1.5%, corr 0.89 with SPY.
|
||||
- Drawdown tables:
|
||||
- ^GSPC 1988-2026, 252-day forward return: 10.3% unconditional, 4.4%
|
||||
from 10-20% off the high, 10.8% from 20-30% off, and 27.9% from more
|
||||
than 30% off. The last bucket is 11 episodes from about 3 bear markets.
|
||||
- 1928-2026, more than 30% off: 3.6% mean, p5 -61%.
|
||||
- NDX more than 30% off: 7.7% vs 17.4% unconditional.
|
||||
- Stocks: results.json averages each bucket over days, and every bucket
|
||||
comes out negative, which is an aggregation artifact. Pooled stock-days
|
||||
(stock_dd_pooled.json; stock_dd_table now reports pooled_excess): names more than 30% off their high beat the
|
||||
member mean by 0.7% over 63 days. That is +3.0% in 2008-09 and 2020
|
||||
and -0.5% on all other days.
|
||||
Interpretation: a short-term index dip is followed by a bounce that is still
|
||||
measurable after publication. Because the rule is in the market so rarely,
|
||||
it is worth something only as a leverage timer, not as a sleeve. A stock
|
||||
dip earns ~20 bp gross per 4-day trade, about what trading it costs. A
|
||||
stock that is deep in its own drawdown rebounds only when the whole market
|
||||
crashed with it. Otherwise it keeps lagging, which is momentum.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from research.lab.factor_scan import RESULTS, load
|
||||
from research.lab.levered_book import stats, tbill
|
||||
from research.lab.us_final import monthly_blend
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
OUT = os.path.join(RESULTS, "dip_buying")
|
||||
END = "2026-09-24"
|
||||
FEE3 = 0.009
|
||||
IDX_COST, IDX_STRESS, STK_COST = 0.0001, 0.0003, 0.001
|
||||
IDX_PERIODS = {"1994-2007": ("1994-01-01", "2007-12-31"), "2008-2026": ("2008-01-01", END),
|
||||
"2017-2026": ("2017-01-01", END)}
|
||||
STK_SPAN = ("2006-10-02", END)
|
||||
STK_GATE = ("2014-01-02", END)
|
||||
BUCKETS = [(0.0, -0.05), (-0.05, -0.10), (-0.10, -0.20), (-0.20, -0.30), (-0.30, -1.01)]
|
||||
|
||||
PREREG = {
|
||||
"question": "Does buying short-term dips (index RSI(2) or trend-filtered stock RSI(2)) earn a "
|
||||
"return worth trading or adding to the three-sleeve book?",
|
||||
"continue_if": {
|
||||
"index (all of)": [
|
||||
"C1 SPY rsi2 next_open 2008-2026: mean net trade return > 0 with t >= 2.0",
|
||||
"C2 SPY rsi2 next_open 2017-2026: mean net trade return > 0 with t >= 1.5",
|
||||
"C3 SPY rsi2 next_open 3x 2008-2026: excess Sharpe >= 0.6 and above SPY buy-and-hold",
|
||||
],
|
||||
"stock (all of)": [
|
||||
"C4 stock rsi2 next_open 2014-2026: excess Sharpe >= 0.8 and above the EW S&P 500 member book",
|
||||
"C5 2014-2026: momentum/lowrisk/trend + stock rsi2 at 1/4 each raises the three-sleeve "
|
||||
"book's excess Sharpe by >= 0.05",
|
||||
],
|
||||
},
|
||||
"decision": {
|
||||
"CONTINUE (index)": "preregister an index-dip leverage overlay; signals from 15:40 Alpaca bars",
|
||||
"CONTINUE (stock)": "register the stock dip sleeve in the web engine and replay it",
|
||||
"STOP": "drop dip buying; the descriptive tables stay as the answer",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def rsi(c, n: int = 2):
|
||||
d = c.diff()
|
||||
up = d.clip(lower=0).ewm(alpha=1 / n, adjust=False).mean()
|
||||
dn = (-d).clip(lower=0).ewm(alpha=1 / n, adjust=False).mean()
|
||||
return 100 - 100 / (1 + up / dn)
|
||||
|
||||
|
||||
def tstat(x) -> float:
|
||||
x = np.asarray(x, float)
|
||||
return float(x.mean() / x.std(ddof=1) * np.sqrt(len(x))) if len(x) > 2 else float("nan")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# A. index dip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def index_rule(c: pd.Series, o: pd.Series, th: float = 10.0, fill: str = "next_open"):
|
||||
"""Per-day in-market return (NaN when flat), trade flags and the trade list."""
|
||||
enter = ((c > c.rolling(200).mean()) & (rsi(c) < th)).to_numpy()
|
||||
leave = (c > c.rolling(5).mean()).to_numpy()
|
||||
cv, ov = c.to_numpy(), o.to_numpy()
|
||||
n = len(c)
|
||||
m = np.full(n, np.nan)
|
||||
traded = np.zeros(n)
|
||||
trades, held, pend, t0, legs = [], False, None, None, []
|
||||
for i in range(1, n):
|
||||
if fill == "next_open":
|
||||
if pend == "buy":
|
||||
held, t0, legs = True, i, [cv[i] / ov[i] - 1]
|
||||
m[i], traded[i] = legs[0], 1
|
||||
elif pend == "sell":
|
||||
m[i], traded[i] = ov[i] / cv[i - 1] - 1, 1
|
||||
legs.append(m[i])
|
||||
trades.append((c.index[t0], c.index[i], float(np.prod(1 + np.array(legs)) - 1), i - t0))
|
||||
held = False
|
||||
elif held:
|
||||
m[i] = cv[i] / cv[i - 1] - 1
|
||||
legs.append(m[i])
|
||||
pend = None
|
||||
if held and leave[i]:
|
||||
pend = "sell"
|
||||
elif not held and enter[i]:
|
||||
pend = "buy"
|
||||
else:
|
||||
if held:
|
||||
m[i] = cv[i] / cv[i - 1] - 1
|
||||
legs.append(m[i])
|
||||
if leave[i]:
|
||||
traded[i] = 1
|
||||
trades.append((c.index[t0], c.index[i], float(np.prod(1 + np.array(legs)) - 1), i - t0))
|
||||
held = False
|
||||
continue
|
||||
if not held and enter[i]:
|
||||
held, t0, legs, traded[i] = True, i, [], 1
|
||||
return pd.Series(m, c.index), pd.Series(traded, c.index), trades
|
||||
|
||||
|
||||
def wrap(m: pd.Series, traded: pd.Series, rf: pd.Series, lev: float, cost: float) -> pd.Series:
|
||||
rfd = rf / 252
|
||||
fee = FEE3 / 252 if lev > 1 else 0.0
|
||||
inside = lev * m - (lev - 1) * rfd - fee
|
||||
return inside.where(m.notna(), rfd) - cost * traded
|
||||
|
||||
|
||||
def trade_stats(trades, a, b, cost) -> dict:
|
||||
r = np.array([x[2] - 2 * cost for x in trades if pd.Timestamp(a) <= x[0] <= pd.Timestamp(b)])
|
||||
days = [x[3] for x in trades if pd.Timestamp(a) <= x[0] <= pd.Timestamp(b)]
|
||||
if len(r) < 3:
|
||||
return {"n": int(len(r))}
|
||||
return {"n": int(len(r)), "win": float((r > 0).mean()), "mean_bp": float(r.mean() * 1e4),
|
||||
"t": tstat(r), "avg_days": float(np.mean(days)), "per_year": float(len(r) / ((pd.Timestamp(b) - pd.Timestamp(a)).days / 365.25))}
|
||||
|
||||
|
||||
def brief(r: pd.Series, rf: pd.Series) -> dict:
|
||||
s = stats(r, rf)
|
||||
return {k: float(s[k]) for k in ("cagr", "sharpe", "vol", "mdd", "worst_yr")}
|
||||
|
||||
|
||||
def part_a(L: dict) -> dict:
|
||||
out = {}
|
||||
for tk in ("SPY", "QQQ"):
|
||||
c = L["close"][tk].dropna().loc[:END]
|
||||
o = L["open"][tk].reindex(c.index)
|
||||
rf = tbill(c.index).fillna(0.0)
|
||||
r_bh = c.pct_change().fillna(0.0)
|
||||
bh3 = 3 * r_bh - 2 * rf / 252 - FEE3 / 252
|
||||
for fill in ("next_open", "close"):
|
||||
m, tr, trades = index_rule(c, o, fill=fill)
|
||||
key = f"{tk}_rsi2_{fill}"
|
||||
rows = {}
|
||||
for per, (a, b) in IDX_PERIODS.items():
|
||||
if c.index[0] > pd.Timestamp(a) - pd.Timedelta(days=300):
|
||||
continue
|
||||
sl = slice(a, b)
|
||||
rows[per] = {
|
||||
"trades": trade_stats(trades, a, b, IDX_COST),
|
||||
"time_in": float(m.loc[sl].notna().mean()),
|
||||
"x1": brief(wrap(m, tr, rf, 1, IDX_COST).loc[sl], rf),
|
||||
"x3": brief(wrap(m, tr, rf, 3, IDX_COST).loc[sl], rf),
|
||||
"x3_stress": brief(wrap(m, tr, rf, 3, IDX_STRESS).loc[sl], rf),
|
||||
"bh": brief(r_bh.loc[sl], rf), "bh3": brief(bh3.loc[sl], rf),
|
||||
# always 1x, 3x inside dips (switch SPY -> UPRO): reported only
|
||||
"bh_plus_dip3": brief((r_bh + (2 * (m - rf / 252) - FEE3 / 252).where(m.notna(), 0.0)
|
||||
- IDX_COST * tr).loc[sl], rf),
|
||||
}
|
||||
out[key] = rows
|
||||
if tk == "SPY" and fill == "next_open":
|
||||
out["_spy_dip_x1"] = wrap(m, tr, rf, 1, IDX_COST)
|
||||
out["_spy_dip_x3"] = wrap(m, tr, rf, 3, IDX_COST)
|
||||
g = L["close"]["^GSPC"].dropna().loc[:"1993-12-31"]
|
||||
m, tr, trades = index_rule(g, g, fill="close")
|
||||
rf = tbill(g.index).fillna(0.0)
|
||||
out["GSPC_rsi2_close_1962-1993"] = {"trades": trade_stats(trades, "1962-01-01", "1993-12-31", 0.0),
|
||||
"time_in": float(m.loc["1962":].notna().mean())}
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# B. stock dip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def stock_rule(p, fill: str = "next_open", slots: int = 10, th: float = 5.0, max_hold: int = 10,
|
||||
cost: float = STK_COST):
|
||||
cf = p.close.ffill(limit=5)
|
||||
enter = p.member & p.close.notna() & (p.close >= 5) & (cf > cf.rolling(200, min_periods=200).mean()) \
|
||||
& (rsi(cf) < th)
|
||||
score = rsi(cf).where(enter).to_numpy()
|
||||
leave = (cf > cf.rolling(5).mean()).to_numpy()
|
||||
C, O, M = p.close.ffill().to_numpy(), p.open.to_numpy(), p.member.to_numpy()
|
||||
raw_c = p.close.to_numpy()
|
||||
cash_r = p.cash_ret.reindex(p.dates).fillna(0.0).to_numpy()
|
||||
n = len(p.dates)
|
||||
cash, pos = 1.0, {} # j -> [shares, entry index, entry value]
|
||||
eq = np.ones(n)
|
||||
pend_sell, pend_buy, trades = set(), [], []
|
||||
stale = {}
|
||||
|
||||
def sell(j, px, i):
|
||||
nonlocal cash
|
||||
sh, t0, v0 = pos.pop(j)
|
||||
cash += sh * px * (1 - cost)
|
||||
trades.append((p.dates[t0], i - t0, sh * px * (1 - cost) / v0 - 1))
|
||||
|
||||
def buy(j, px, budget, i):
|
||||
nonlocal cash
|
||||
amt = min(budget, cash)
|
||||
if amt < 0.5 * budget or not np.isfinite(px) or px <= 0:
|
||||
return
|
||||
cash -= amt
|
||||
pos[j] = [amt / (px * (1 + cost)), i, amt]
|
||||
|
||||
for i in range(1, n):
|
||||
cash *= 1 + cash_r[i]
|
||||
if fill == "next_open":
|
||||
for j in list(pend_sell):
|
||||
if np.isfinite(O[i, j]):
|
||||
sell(j, O[i, j], i)
|
||||
pend_sell.discard(j)
|
||||
budget = eq[i - 1] / slots
|
||||
for j in pend_buy:
|
||||
if j not in pos and len(pos) < slots:
|
||||
buy(j, O[i, j], budget, i)
|
||||
pend_buy = []
|
||||
for j in list(pos): # settle names with no print for 5 sessions
|
||||
stale[j] = 0 if np.isfinite(raw_c[i, j]) else stale.get(j, 0) + 1
|
||||
if stale[j] >= 5:
|
||||
sell(j, C[i, j], i)
|
||||
pend_sell.discard(j)
|
||||
val = cash + sum(sh * C[i, j] for j, (sh, _, _) in pos.items())
|
||||
exits = [j for j, (_, t0, _) in pos.items()
|
||||
if leave[i, j] or i - t0 >= max_hold or not M[i, j]]
|
||||
if fill == "close":
|
||||
for j in exits:
|
||||
if np.isfinite(raw_c[i, j]):
|
||||
sell(j, raw_c[i, j], i)
|
||||
else:
|
||||
pend_sell.update(exits)
|
||||
free = slots - (len(pos) - len(pend_sell & set(pos)))
|
||||
row = score[i]
|
||||
cand = np.where(np.isfinite(row))[0]
|
||||
cand = [j for j in cand[np.argsort(row[cand])] if j not in pos][:max(free, 0)]
|
||||
if fill == "close":
|
||||
for j in cand:
|
||||
buy(j, raw_c[i, j], val / slots, i)
|
||||
else:
|
||||
pend_buy = cand
|
||||
eq[i] = cash + sum(sh * C[i, j] for j, (sh, _, _) in pos.items())
|
||||
eqs = pd.Series(eq, p.dates)
|
||||
return eqs.pct_change().fillna(0.0), trades
|
||||
|
||||
|
||||
def part_b(p, sleeves: pd.DataFrame) -> dict:
|
||||
from research.lab.us_sweep import run
|
||||
sp = p.slice("2005-06-01", END)
|
||||
rf = tbill(sp.dates).fillna(0.0)
|
||||
out, R = {}, {}
|
||||
for fill in ("next_open", "close"):
|
||||
r, trades = stock_rule(sp, fill=fill)
|
||||
R[f"stock_rsi2_{fill}"] = r.loc[STK_SPAN[0]:]
|
||||
rows = {}
|
||||
for per, (a, b) in {"2006-2013": (STK_SPAN[0], "2013-12-31"), "2014-2026": STK_GATE}.items():
|
||||
ts = [x for x in trades if pd.Timestamp(a) <= x[0] <= pd.Timestamp(b)]
|
||||
net = np.array([x[2] for x in ts])
|
||||
gross = net + 2 * STK_COST
|
||||
rows[per] = {**brief(r.loc[a:b], rf), "n_trades": len(ts), "win": float((net > 0).mean()),
|
||||
"mean_net_bp": float(net.mean() * 1e4), "t": tstat(net),
|
||||
"breakeven_bp_side": float(gross.mean() / 2 * 1e4),
|
||||
"avg_days": float(np.mean([x[1] for x in ts]))}
|
||||
out[f"stock_rsi2_{fill}"] = rows
|
||||
c = sp.close.ffill(limit=5)
|
||||
mr = (-(c / c.shift(21) - 1)).where((c / c.shift(252) - 1 > 0) & sp.member)
|
||||
st = run(sp, mr, [(STK_SPAN[0], "2013-12-31"), STK_GATE], top_n=20, buffer=0.0, every=21)
|
||||
R["mean_reversion_monthly"] = st["_rets"]
|
||||
out["mean_reversion_monthly"] = {"2006-2013": brief(st["_rets"].loc[:"2013-12-31"], rf),
|
||||
"2014-2026": brief(st["_rets"].loc[STK_GATE[0]:], rf),
|
||||
"turn": float(st["turn"])}
|
||||
R = pd.DataFrame(R)
|
||||
S = sleeves.join(R, how="left")
|
||||
gate = S.loc[STK_GATE[0]:STK_GATE[1]]
|
||||
rfg = tbill(gate.index).fillna(0.0)
|
||||
out["ew_sp500_2014-2026"] = brief(gate["ew_sp500"], rfg)
|
||||
out["spy_2014-2026"] = brief(gate["spy"], rfg)
|
||||
base = gate[["momentum", "lowrisk", "trend"]].fillna(0.0)
|
||||
three = monthly_blend(base, {"momentum": 1 / 3, "lowrisk": 1 / 3, "trend": 1 / 3})
|
||||
out["book3_2014-2026"] = brief(three, rfg)
|
||||
mon = lambda x: (1 + x).resample("ME").prod() - 1
|
||||
for k in R.columns:
|
||||
four = monthly_blend(gate[["momentum", "lowrisk", "trend", k]].fillna(0.0),
|
||||
{"momentum": .25, "lowrisk": .25, "trend": .25, k: .25})
|
||||
out[f"book4_{k}_2014-2026"] = brief(four, rfg)
|
||||
out[f"corr_{k}"] = {s: float(mon(gate[k]).corr(mon(gate[s]))) for s in ("momentum", "lowrisk", "trend", "spy")}
|
||||
out["_R"] = R
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# C. descriptive drawdown tables
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def bucket_of(dd: pd.DataFrame | pd.Series):
|
||||
lab = dd * np.nan
|
||||
for k, (hi, lo) in enumerate(BUCKETS):
|
||||
lab = lab.mask((dd <= hi) & (dd > lo), k)
|
||||
return lab
|
||||
|
||||
|
||||
def index_dd_table(c: pd.Series, a: str, b: str) -> list[dict]:
|
||||
c = c.dropna()
|
||||
dd = c / c.rolling(252, min_periods=200).max() - 1
|
||||
lab = bucket_of(dd).loc[a:b]
|
||||
rows = []
|
||||
for k, (hi, lo) in enumerate(BUCKETS + [(None, None)]):
|
||||
sel = lab.notna() if hi is None else lab == k
|
||||
row = {"bucket": "all" if hi is None else f"{-hi:.0%}..{-lo if lo > -1 else 1:.0%}",
|
||||
"share_days": float(sel.mean()),
|
||||
"episodes": int((sel & ~sel.shift(1, fill_value=False)).sum())}
|
||||
for h in (21, 63, 252):
|
||||
f = (c.shift(-h) / c - 1).loc[a:b][sel].dropna()
|
||||
row[f"fwd{h}"] = {"mean": float(f.mean()), "median": float(f.median()), "p_up": float((f > 0).mean()),
|
||||
"p5": float(f.quantile(0.05))} if len(f) else {}
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def stock_dd_table(p, a: str, b: str) -> list[dict]:
|
||||
c = p.close.ffill(limit=5)
|
||||
dd = c / c.rolling(252, min_periods=200).max() - 1
|
||||
lab = bucket_of(dd).where(p.member & (p.close >= 5))
|
||||
later = p.close.ffill() # a delisted name keeps its last print
|
||||
entry = p.open.shift(-1)
|
||||
rows = []
|
||||
for h in (21, 63):
|
||||
fwd = later.shift(-h) / entry - 1
|
||||
fwd = fwd.where(lab.notna())
|
||||
ex = fwd.sub(fwd.mean(axis=1), axis=0)
|
||||
for k, (hi, lo) in enumerate(BUCKETS):
|
||||
daily = ex.where(lab == k).mean(axis=1).loc[a:b].dropna()
|
||||
spaced = daily.iloc[::h] # non-overlapping for the t-stat
|
||||
rows.append({"h": h, "bucket": f"{-hi:.0%}..{-lo if lo > -1 else 1:.0%}",
|
||||
"share": float((lab == k).loc[a:b].sum().sum() / lab.loc[a:b].notna().sum().sum()),
|
||||
"excess_mean": float(daily.mean()), "t_nonoverlap": tstat(spaced),
|
||||
"pooled_excess": float(ex.where(lab == k).loc[a:b].stack().mean()),
|
||||
"p_beats_mean": float((ex.where(lab == k).loc[a:b].stack() > 0).mean())})
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
res_path = os.path.join(OUT, "results.json")
|
||||
if os.path.exists(res_path):
|
||||
raise SystemExit(f"{res_path} exists; this study does not overwrite results")
|
||||
with open(os.path.join(OUT, "prereg.json"), "w") as fh:
|
||||
json.dump(PREREG, fh, indent=2)
|
||||
|
||||
with open(os.path.join("data", "research", "long_history.pkl"), "rb") as fh:
|
||||
L = pickle.load(fh)
|
||||
res = {"index": part_a(L)}
|
||||
spy_dip = pd.DataFrame({"spy_dip_x1": res["index"].pop("_spy_dip_x1"),
|
||||
"spy_dip_x3": res["index"].pop("_spy_dip_x3")})
|
||||
for k, rows in res["index"].items():
|
||||
if k.startswith("_"):
|
||||
continue
|
||||
if "trades" in rows:
|
||||
print(f"{k}: {rows['trades']} time in {rows['time_in']:.1%}")
|
||||
continue
|
||||
for per, r in rows.items():
|
||||
t = r["trades"]
|
||||
print(f"{k:22s} {per}: n {t['n']:3d} ({t.get('per_year', 0):.1f}/yr) win {t.get('win', 0):.0%} "
|
||||
f"{t.get('mean_bp', 0):6.1f} bp/trade t {t.get('t', 0):4.2f} days {t.get('avg_days', 0):.1f} "
|
||||
f"in {r['time_in']:.0%}")
|
||||
for s in ("x1", "x3", "x3_stress", "bh", "bh3", "bh_plus_dip3"):
|
||||
x = r[s]
|
||||
print(f" {s:13s} cagr {x['cagr']*100:5.1f}% xsh {x['sharpe']:5.2f} vol {x['vol']*100:4.1f}% "
|
||||
f"mdd {x['mdd']*100:6.1f}% worst yr {x['worst_yr']*100:6.1f}%")
|
||||
|
||||
res["index_dd"] = {"GSPC_1928-2026": index_dd_table(L["close"]["^GSPC"], "1928-12-31", END),
|
||||
"GSPC_1988-2026": index_dd_table(L["close"]["^GSPC"], "1988-01-01", END),
|
||||
"NDX_1986-2026": index_dd_table(L["close"]["^NDX"], "1986-10-01", END)}
|
||||
for k, rows in res["index_dd"].items():
|
||||
print(f"\n{k}: forward return by drawdown from the 252-day high")
|
||||
for r in rows:
|
||||
f = " ".join(f"{h}d {r[f'fwd{h}']['mean']*100:6.1f}%/{r[f'fwd{h}']['median']*100:5.1f}% "
|
||||
f"up {r[f'fwd{h}']['p_up']:.0%} p5 {r[f'fwd{h}']['p5']*100:6.1f}%"
|
||||
for h in (21, 63, 252) if r[f"fwd{h}"])
|
||||
print(f" {r['bucket']:>9s} days {r['share_days']:5.1%} ep {r['episodes']:4d} | {f}")
|
||||
|
||||
p, _ = load("us")
|
||||
sleeves = pd.read_pickle(os.path.join(RESULTS, "us_final_sleeves.pkl"))
|
||||
b = part_b(p, sleeves)
|
||||
R = b.pop("_R")
|
||||
res["stock"] = b
|
||||
for k, v in b.items():
|
||||
print(f"{k}: {json.dumps(v, default=float)}")
|
||||
res["stock_dd"] = {"2006-2026": stock_dd_table(p, STK_SPAN[0], END), "2014-2026": stock_dd_table(p, *STK_GATE)}
|
||||
for k, rows in res["stock_dd"].items():
|
||||
print(f"\nstocks {k}: excess return over the member mean by drawdown from the 252-day high")
|
||||
for r in rows:
|
||||
print(f" h{r['h']:2d} {r['bucket']:>9s} share {r['share']:5.1%} excess {r['excess_mean']*100:6.2f}% "
|
||||
f"t {r['t_nonoverlap']:5.2f} beats mean {r['p_beats_mean']:.0%}")
|
||||
|
||||
spy = res["index"]["SPY_rsi2_next_open"]
|
||||
gate = b["stock_rsi2_next_open"]["2014-2026"]
|
||||
checks = {
|
||||
"C1": spy["2008-2026"]["trades"]["mean_bp"] > 0 and spy["2008-2026"]["trades"]["t"] >= 2.0,
|
||||
"C2": spy["2017-2026"]["trades"]["mean_bp"] > 0 and spy["2017-2026"]["trades"]["t"] >= 1.5,
|
||||
"C3": spy["2008-2026"]["x3"]["sharpe"] >= 0.6 and spy["2008-2026"]["x3"]["sharpe"] > spy["2008-2026"]["bh"]["sharpe"],
|
||||
"C4": gate["sharpe"] >= 0.8 and gate["sharpe"] > b["ew_sp500_2014-2026"]["sharpe"],
|
||||
"C5": b["book4_stock_rsi2_next_open_2014-2026"]["sharpe"] >= b["book3_2014-2026"]["sharpe"] + 0.05,
|
||||
}
|
||||
idx_ok = checks["C1"] and checks["C2"] and checks["C3"]
|
||||
stk_ok = checks["C4"] and checks["C5"]
|
||||
decision = "CONTINUE (index)" if idx_ok else ""
|
||||
decision += (" + " if decision and stk_ok else "") + ("CONTINUE (stock)" if stk_ok else "")
|
||||
res["verdict"] = {"checks": checks, "decision": decision or "STOP"}
|
||||
print(f"\nchecks {checks} -> {res['verdict']['decision']}")
|
||||
pd.concat([R, spy_dip], axis=1).to_pickle(os.path.join(OUT, "returns.pkl"))
|
||||
with open(res_path, "w") as fh:
|
||||
json.dump(res, fh, indent=2, default=float)
|
||||
from research.lab.trials import append_trials
|
||||
rows = [{"name": f"dip_buying:{k}:{per}:{s}", **v[per][s]} for k, v in res["index"].items()
|
||||
if "trades" not in v for per in v for s in ("x1", "x3")]
|
||||
rows += [{"name": f"dip_buying:{k}:{per}", **{kk: vv for kk, vv in v[per].items()}}
|
||||
for k, v in b.items() if k.startswith("stock_rsi2") for per in v]
|
||||
append_trials(os.path.join(RESULTS, "trials_us.jsonl"), rows,
|
||||
{"study": "dip_buying", "role": "preregistered quick check"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
341
research/lab/dip_leverage_overlay.py
Normal file
341
research/lab/dip_leverage_overlay.py
Normal file
@@ -0,0 +1,341 @@
|
||||
"""
|
||||
Does adding index exposure during short-term SPY dips improve the US books?
|
||||
research/lab/dip_buying.py found that the SPY RSI(2) dip is real (46 bp a
|
||||
trade, t 3.4, 2008-26) but is in the market on only 14% of days. Its
|
||||
reported-only row "SPY 1x, 3x inside dips" raised SPY's Sharpe from 0.57 to
|
||||
0.69. This study puts that overlay on top of the books instead of SPY.
|
||||
|
||||
PYTHONPATH=. uv run python -m research.lab.dip_leverage_overlay
|
||||
|
||||
Books (daily returns, costs inside; research/lab/overnight_book.py books.pkl):
|
||||
three_sleeve_v7 momentum + lowrisk + trend_v7 (the live us_three_sleeve) gated
|
||||
three_sleeve_overnight overnight-only momentum + lowrisk + trend_v7 (paper test) gated
|
||||
two_sleeve_overnight_trend reported
|
||||
|
||||
Signal: dip_buying.index_rule on SPY. Enter when the close is above its
|
||||
200-day mean and RSI(2) < 10. Exit at the first close above the 5-day mean.
|
||||
|
||||
Overlay (gated: k = 1, next_open fills): book + k x SPY from the open after
|
||||
the signal close to the open after the exit close, as an MES / margin
|
||||
position. It is financed at T-bill + 1% for every day touched, including
|
||||
both partial days, and pays 1 bp a side on k.
|
||||
reported k = 0.5 and 2; close fills; financing at T-bill + 3%; QQQ
|
||||
signal and instrument; "book scale", which levers the whole
|
||||
book (1 + k) inside the windows, with 10 bp a side and either
|
||||
close timing (optimistic) or only full days inside next_open
|
||||
windows (conservative)
|
||||
baselines - the same average SPY exposure held all the time (no timing)
|
||||
- P1: the same windows shifted circularly by a random offset
|
||||
- P2 (gated): the same windows at random non-overlapping
|
||||
starts whose signal day also has SPY above its 200-day
|
||||
mean. P2 separates the dip timing from the trend filter.
|
||||
500 draws each, seed 20261012.
|
||||
|
||||
Span 2006-10-02 .. 2026-09-24, halves split at 2017 (overnight_book.HALVES).
|
||||
All of it was inspected before, and the SPY version of this overlay was seen
|
||||
on 2008-26 before this rule was written. This is a consistency check on the
|
||||
books, not out-of-sample evidence. The rule is written to prereg.json before
|
||||
the run; the script refuses to overwrite results.json.
|
||||
|
||||
Result (2026-10-12): STOP. No gated book passes. three_sleeve_v7 fails D1
|
||||
and D3; three_sleeve_overnight fails D1, D2 and D3.
|
||||
|
||||
The overlay alone (k = 1): 160 windows, 14% of days, +3.2%/yr, vol 6.4%,
|
||||
self-financed Sharpe 0.50, positive in 15 of 21 years (2011 and 2018 -6.7%).
|
||||
|
||||
book Sharpe + SPY k1 (halves) MaxDD P2 rank constant SPY
|
||||
three_sleeve_v7 0.98 0.99 (0.75 / 1.28) -37.2 -> -37.2% 88% 0.94
|
||||
three_sleeve_overnight 1.38 1.33 (1.14 / 1.55) -16.5 -> -20.4% 77% 1.31
|
||||
two_sleeve_overnight_trend 1.39 1.38 (1.13 / 1.64) -20.7 -> -20.7% 84% 1.36
|
||||
|
||||
- k = 1 adds about 3.3pp of CAGR to each book, but vol rises by the same
|
||||
proportion, so the Sharpe ratio does not move. k = 2 lowers it (0.94 /
|
||||
1.20 / 1.29). k = 0.5 is flat. Close fills give +0.04 / -0.01 / +0.02.
|
||||
Financing at +3% costs 0.02. A QQQ signal and instrument is worse.
|
||||
- Dip timing beats random trend-up timing. The P2 placebo gives -0.05 to
|
||||
-0.10 Sharpe; the real gain is -0.05 to +0.01. That ranks 77-88%, under
|
||||
the 95% bar. It also beats holding the same average SPY exposure all the
|
||||
time (+0.02 to +0.05 Sharpe, smaller MaxDD) because the 200-day filter
|
||||
keeps it out of bear markets.
|
||||
- Why it adds nothing: the books already earn the bounce. Inside the
|
||||
windows the books earn 50-64%/yr, against 13-22% outside. Their
|
||||
correlation with the SPY leg there is 0.57-0.75, and they move about
|
||||
2x SPY, through TrendRider's 3x ETFs and high-beta momentum names. A
|
||||
Sharpe-0.5 leg correlated with a Sharpe-1.4 book lowers the book's
|
||||
Sharpe.
|
||||
- Book scale (reported) helps only the weakest book, and only with close
|
||||
timing: v7 0.98 -> 1.04. With full days only it is 0.95. Overnight:
|
||||
1.36 / 1.29.
|
||||
- Leverage for 50%/yr:
|
||||
three_sleeve_overnight 2.35x, MaxDD -36.7%, worst yr -18.6%
|
||||
with the overlay 2.02x, -38.4%, -9.3%
|
||||
two_sleeve_overnight_trend 1.73x, -34.8%, -24.3%
|
||||
with the overlay 1.54x, -31.6%, -16.8%
|
||||
three_sleeve_v7 about 2.9x, -79% (grid check after the
|
||||
run), against 3.6x, -86% without it
|
||||
The overlay swaps one kind of leverage for another, at about the same
|
||||
risk.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from research.lab.dip_buying import index_rule
|
||||
from research.lab.factor_scan import RESULTS
|
||||
from research.lab.levered_book import stats, tbill
|
||||
from research.lab.overnight_book import HALVES, SPAN, leverage_for, levered
|
||||
|
||||
OUT = os.path.join(RESULTS, "dip_leverage_overlay")
|
||||
BOOKS_PKL = os.path.join(RESULTS, "overnight_book", "books.pkl")
|
||||
GATED = ["three_sleeve_v7", "three_sleeve_overnight"]
|
||||
REPORTED = ["two_sleeve_overnight_trend"]
|
||||
K = 1.0
|
||||
SPREAD, SPREAD_STRESS = 0.01, 0.03
|
||||
IDX_COST, BOOK_COST = 0.0001, 0.001
|
||||
N_PLACEBO, SEED = 500, 20261012
|
||||
|
||||
PREREG = {
|
||||
"question": "Does +k x SPY held only inside SPY RSI(2) dip windows raise a book's Sharpe, beyond what "
|
||||
"the same exposure at random trend-up times gives?",
|
||||
"span": SPAN, "halves": HALVES, "gated_books": GATED, "reported_books": REPORTED,
|
||||
"overlay": "k = 1, next_open fills, financing T-bill + 1% on every day touched, 1 bp a side",
|
||||
"gates_per_book": [
|
||||
"D1 excess Sharpe (book + overlay) >= book + 0.05 over the span, and higher than the book in each half",
|
||||
"D2 MaxDD (book + overlay) >= book MaxDD - 3pp",
|
||||
"D3 the span Sharpe gain > 95th percentile of P2 (trend-matched random windows, 500 draws)",
|
||||
],
|
||||
"decision": {
|
||||
"CONTINUE": "a gated book passes D1-D3 -> paper-trade the overlay alongside that book from a "
|
||||
"15:40 signal (Alpaca 1-min SPY bars), then decide on live use",
|
||||
"STOP": "no gated book passes -> do not add a dip overlay; leverage stays static or vol-managed",
|
||||
},
|
||||
"reported": "k 0.5/2, close fills, financing stress, QQQ, book scale, constant-exposure baseline, P1, "
|
||||
"leverage for 50%/yr with and without the overlay, yearly overlay P&L, book return inside "
|
||||
"vs outside the windows",
|
||||
"data_status": "2006-2026 previously inspected; the SPY overlay result was seen before this rule",
|
||||
}
|
||||
|
||||
|
||||
def windows(c: pd.Series, o: pd.Series, fill: str) -> list[tuple[int, int]]:
|
||||
"""(a, b) positions: next_open holds from the open of a to the open of b;
|
||||
close holds from the close of a to the close of b."""
|
||||
_, _, trades = index_rule(c, o, fill=fill)
|
||||
pos = {d: i for i, d in enumerate(c.index)}
|
||||
return [(pos[t[0]], pos[t[1]]) for t in trades]
|
||||
|
||||
|
||||
def overlay_parts(cv: np.ndarray, ov: np.ndarray, wins, fill: str):
|
||||
"""In-market index return, financed-day flag and trade count per day."""
|
||||
n = len(cv)
|
||||
rcc = np.r_[0.0, cv[1:] / cv[:-1] - 1]
|
||||
m, inside, tr = np.zeros(n), np.zeros(n), np.zeros(n)
|
||||
for a, b in wins:
|
||||
if fill == "next_open":
|
||||
m[a] += cv[a] / ov[a] - 1
|
||||
m[a + 1:b] += rcc[a + 1:b]
|
||||
m[b] += ov[b] / cv[b - 1] - 1
|
||||
inside[a:b + 1] = 1
|
||||
else:
|
||||
m[a + 1:b + 1] += rcc[a + 1:b + 1]
|
||||
inside[a + 1:b + 1] = 1
|
||||
tr[a] += 1
|
||||
tr[b] += 1
|
||||
return m, inside, tr
|
||||
|
||||
|
||||
def overlay(parts, rfd: np.ndarray, k: float, spread: float = SPREAD, cost: float = IDX_COST) -> np.ndarray:
|
||||
m, inside, tr = parts
|
||||
return k * (m - inside * (rfd + spread / 252)) - k * cost * tr
|
||||
|
||||
|
||||
def xsh(r: np.ndarray, rfd: np.ndarray) -> float:
|
||||
return float((r - rfd).mean() / r.std() * np.sqrt(252))
|
||||
|
||||
|
||||
def summary(r: pd.Series, rf: pd.Series) -> dict:
|
||||
s = stats(r, rf)
|
||||
d = {k: float(s[k]) for k in ("cagr", "sharpe", "vol", "mdd", "worst_yr")}
|
||||
d["halves"] = [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES]
|
||||
return d
|
||||
|
||||
|
||||
def placebo_trend(rng, wins, elig, lo, hi):
|
||||
"""Same window lengths at random non-overlapping starts with a trend-up signal day."""
|
||||
cand = np.where(elig[lo:hi - 1])[0] + lo
|
||||
taken = np.zeros(hi + 2, bool)
|
||||
out = []
|
||||
for D in rng.permutation([b - a for a, b in wins]):
|
||||
for _ in range(200):
|
||||
a = int(rng.choice(cand)) + 1
|
||||
b = a + D
|
||||
if b >= hi or taken[max(a - 1, 0):b + 2].any():
|
||||
continue
|
||||
taken[a:b + 1] = True
|
||||
out.append((a, b))
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def placebo_shift(rng, wins, lo, hi):
|
||||
s = int(rng.integers(63, hi - lo - 63))
|
||||
out = []
|
||||
for a, b in wins:
|
||||
a2 = lo + (a - lo + s) % (hi - lo)
|
||||
b2 = a2 + (b - a)
|
||||
if b2 < hi:
|
||||
out.append((a2, b2))
|
||||
return out
|
||||
|
||||
|
||||
def fmt(name: str, d: dict) -> str:
|
||||
return (f"{name:46s} cagr {d['cagr']*100:5.1f}% xsh {d['sharpe']:5.2f} vol {d['vol']*100:4.1f}% "
|
||||
f"mdd {d['mdd']*100:6.1f}% worst yr {d['worst_yr']*100:6.1f}% halves "
|
||||
f"{d['halves'][0]:5.2f} {d['halves'][1]:5.2f}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
res_path = os.path.join(OUT, "results.json")
|
||||
if os.path.exists(res_path):
|
||||
raise SystemExit(f"{res_path} exists; this study does not overwrite results")
|
||||
with open(os.path.join(OUT, "prereg.json"), "w") as fh:
|
||||
json.dump(PREREG, fh, indent=2)
|
||||
|
||||
B = pd.read_pickle(BOOKS_PKL).loc[SPAN[0]:SPAN[1]]
|
||||
idx = B.index
|
||||
with open(os.path.join("data", "research", "long_history.pkl"), "rb") as fh:
|
||||
L = pickle.load(fh)
|
||||
rf = tbill(idx).fillna(0.0)
|
||||
rfd = (rf / 252).to_numpy()
|
||||
|
||||
inst = {}
|
||||
for tk in ("SPY", "QQQ"):
|
||||
c = L["close"][tk].dropna().loc[:SPAN[1]]
|
||||
o = L["open"][tk].reindex(c.index)
|
||||
miss = idx.difference(c.index)
|
||||
assert len(miss) == 0, f"{tk} lacks {len(miss)} book dates"
|
||||
sma = c.rolling(200).mean()
|
||||
# windows on the full history, then positions in the book's calendar
|
||||
on_idx = lambda w: [(idx.get_loc(c.index[a]), idx.get_loc(c.index[b])) for a, b in w
|
||||
if c.index[a] >= idx[0] and c.index[b] in idx]
|
||||
cv, ov = c.reindex(idx).to_numpy(), o.reindex(idx).to_numpy()
|
||||
inst[tk] = {"cv": cv, "ov": ov, "elig": (c > sma).reindex(idx).fillna(False).to_numpy(),
|
||||
"w": {f: on_idx(windows(c, o, f)) for f in ("next_open", "close")}}
|
||||
spy = inst["SPY"]
|
||||
parts = {f: overlay_parts(spy["cv"], spy["ov"], spy["w"][f], f) for f in ("next_open", "close")}
|
||||
ov1 = overlay(parts["next_open"], rfd, K)
|
||||
share_in = float(parts["next_open"][1].mean())
|
||||
|
||||
res = {"n_windows": len(spy["w"]["next_open"]), "share_days": share_in, "overlay": {}, "books": {}}
|
||||
ovs = pd.Series(ov1, idx)
|
||||
res["overlay"] = {"ann_mean": float(ovs.mean() * 252), "vol": float(ovs.std() * np.sqrt(252)),
|
||||
"sharpe_self_financed": float(ovs.mean() / ovs.std() * np.sqrt(252)),
|
||||
"yearly": {int(y): float(v) for y, v in ovs.groupby(idx.year).sum().items()}}
|
||||
yrs = pd.Series(res["overlay"]["yearly"])
|
||||
print(f"windows {res['n_windows']}, share of days {share_in:.1%}; overlay k=1 mean {res['overlay']['ann_mean']*100:.2f}%/yr "
|
||||
f"vol {res['overlay']['vol']*100:.1f}% self-financed Sharpe {res['overlay']['sharpe_self_financed']:.2f}; "
|
||||
f"positive years {(yrs > 0).sum()}/{len(yrs)}")
|
||||
print("overlay P&L by year %:", (yrs * 100).round(1).to_dict())
|
||||
|
||||
rng = np.random.default_rng(SEED)
|
||||
lo, hi = 0, len(idx)
|
||||
P2 = [overlay(overlay_parts(spy["cv"], spy["ov"], placebo_trend(rng, spy["w"]["next_open"], spy["elig"], lo, hi),
|
||||
"next_open"), rfd, K) for _ in range(N_PLACEBO)]
|
||||
P1 = [overlay(overlay_parts(spy["cv"], spy["ov"], placebo_shift(rng, spy["w"]["next_open"], lo, hi),
|
||||
"next_open"), rfd, K) for _ in range(N_PLACEBO)]
|
||||
m_in = parts["next_open"][1] > 0
|
||||
r_spy_cc = np.r_[0.0, spy["cv"][1:] / spy["cv"][:-1] - 1]
|
||||
const = share_in * K * (r_spy_cc - (rfd + SPREAD / 252))
|
||||
|
||||
gates = {}
|
||||
for name in GATED + REPORTED:
|
||||
r = B[name].to_numpy()
|
||||
base = summary(B[name], rf)
|
||||
row = {"book": base, "variants": {}}
|
||||
print(f"\n=== {name}")
|
||||
print(fmt("book", base))
|
||||
variants = {
|
||||
"spy k1 next_open": r + ov1,
|
||||
"spy k0.5 next_open": r + overlay(parts["next_open"], rfd, 0.5),
|
||||
"spy k2 next_open": r + overlay(parts["next_open"], rfd, 2.0),
|
||||
"spy k1 close": r + overlay(parts["close"], rfd, K),
|
||||
"spy k1 next_open fin+3%": r + overlay(parts["next_open"], rfd, K, SPREAD_STRESS),
|
||||
"constant SPY, same avg exposure": r + const,
|
||||
}
|
||||
q = inst["QQQ"]
|
||||
variants["qqq k1 next_open"] = r + overlay(overlay_parts(q["cv"], q["ov"], q["w"]["next_open"], "next_open"),
|
||||
rfd, K)
|
||||
# book scale: lever the book itself inside the windows
|
||||
_, in_close, tr_close = parts["close"]
|
||||
variants["book scale k1 close timing"] = (r * (1 + K * in_close) - K * in_close * (rfd + SPREAD / 252)
|
||||
- K * BOOK_COST * tr_close)
|
||||
full_days = np.zeros(len(r))
|
||||
for a, b in spy["w"]["next_open"]:
|
||||
full_days[a + 1:b] = 1
|
||||
variants["book scale k1 full days only"] = (r * (1 + K * full_days) - K * parts["next_open"][1] * (rfd + SPREAD / 252)
|
||||
- K * BOOK_COST * parts["next_open"][2])
|
||||
for vname, x in variants.items():
|
||||
d = summary(pd.Series(x, idx), rf)
|
||||
row["variants"][vname] = d
|
||||
print(fmt(" " + vname, d))
|
||||
s0 = xsh(r, rfd)
|
||||
g2 = np.array([xsh(r + p, rfd) - s0 for p in P2])
|
||||
g1 = np.array([xsh(r + p, rfd) - s0 for p in P1])
|
||||
gain = row["variants"]["spy k1 next_open"]["sharpe"] - base["sharpe"]
|
||||
row["placebo"] = {"gain": gain, "P2_mean": float(g2.mean()), "P2_p95": float(np.quantile(g2, 0.95)),
|
||||
"P2_pct_rank": float((g2 < gain).mean()), "P1_mean": float(g1.mean()),
|
||||
"P1_p95": float(np.quantile(g1, 0.95)), "P1_pct_rank": float((g1 < gain).mean())}
|
||||
print(f" Sharpe gain {gain:+.3f} | P2 trend-matched mean {g2.mean():+.3f} p95 {np.quantile(g2, .95):+.3f} "
|
||||
f"rank {row['placebo']['P2_pct_rank']:.1%} | P1 shift mean {g1.mean():+.3f} p95 {np.quantile(g1, .95):+.3f} "
|
||||
f"rank {row['placebo']['P1_pct_rank']:.1%}")
|
||||
ins, outs = r[m_in], r[~m_in]
|
||||
row["book_inside_vs_outside"] = {"inside_ann": float(ins.mean() * 252), "outside_ann": float(outs.mean() * 252),
|
||||
"inside_vol": float(ins.std() * np.sqrt(252)),
|
||||
"outside_vol": float(outs.std() * np.sqrt(252)),
|
||||
"corr_with_overlay_inside": float(np.corrcoef(ins, ov1[m_in])[0, 1])}
|
||||
bi = row["book_inside_vs_outside"]
|
||||
print(f" book inside windows {bi['inside_ann']*100:5.1f}%/yr (vol {bi['inside_vol']*100:4.1f}%) vs outside "
|
||||
f"{bi['outside_ann']*100:5.1f}%/yr (vol {bi['outside_vol']*100:4.1f}%); corr with SPY leg inside "
|
||||
f"{bi['corr_with_overlay_inside']:.2f}")
|
||||
lev = {}
|
||||
for lname, x in (("book", B[name]), ("book + spy k1", pd.Series(r + ov1, idx))):
|
||||
Lx = leverage_for(x, rf)
|
||||
d = {"L": Lx}
|
||||
if np.isfinite(Lx):
|
||||
s = stats(levered(x, rf, Lx), rf)
|
||||
d.update({"mdd": float(s["mdd"]), "worst_yr": float(s["worst_yr"]), "vol": float(s["vol"]),
|
||||
"halves_cagr": [float(stats(levered(x.loc[a:b], rf, Lx), rf)["cagr"]) for a, b in HALVES]})
|
||||
print(f" 50%/yr: {lname:14s} L {Lx:4.2f} mdd {d['mdd']*100:6.1f}% worst yr {d['worst_yr']*100:6.1f}% "
|
||||
f"halves cagr {d['halves_cagr'][0]*100:5.1f}% {d['halves_cagr'][1]*100:5.1f}%")
|
||||
lev[lname] = d
|
||||
row["leverage_for_50"] = lev
|
||||
res["books"][name] = row
|
||||
if name in GATED:
|
||||
v = row["variants"]["spy k1 next_open"]
|
||||
gates[name] = {
|
||||
"D1": v["sharpe"] >= base["sharpe"] + 0.05 and all(v["halves"][i] > base["halves"][i] for i in range(2)),
|
||||
"D2": v["mdd"] >= base["mdd"] - 0.03,
|
||||
"D3": gain > row["placebo"]["P2_p95"],
|
||||
}
|
||||
passing = [k for k, g in gates.items() if all(g.values())]
|
||||
res["verdict"] = {"gates": gates, "passing": passing, "decision": "CONTINUE" if passing else "STOP"}
|
||||
print(f"\ngates {gates}\n-> {res['verdict']['decision']} {passing}")
|
||||
with open(res_path, "w") as fh:
|
||||
json.dump(res, fh, indent=2)
|
||||
pd.DataFrame({"overlay_spy_k1": ov1, "inside": parts["next_open"][1]}, index=idx).to_pickle(
|
||||
os.path.join(OUT, "overlay.pkl"))
|
||||
from research.lab.trials import append_trials
|
||||
append_trials(os.path.join(RESULTS, "trials_us.jsonl"),
|
||||
[{"name": f"dip_leverage_overlay:{b}:{v}", **{x: y for x, y in d.items() if x != "halves"}}
|
||||
for b, row in res["books"].items() for v, d in row["variants"].items()],
|
||||
{"study": "dip_leverage_overlay", "role": "preregistered consistency check"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
362
research/lab/factor_census.py
Normal file
362
research/lab/factor_census.py
Normal file
@@ -0,0 +1,362 @@
|
||||
"""
|
||||
Factor census: every stock factor the lab has, measured one way on the
|
||||
current US panel (PIT S&P 500 after the Tiingo merge and the pre-2011
|
||||
membership fix). Earlier per-factor numbers (factor_scan_us.csv,
|
||||
qqq_scan_factors.csv, ranks_us.pkl) predate that panel.
|
||||
|
||||
PYTHONPATH=. uv run --with numba python -m research.lab.factor_census
|
||||
|
||||
Factors: every research/lab/factors.py factor for "us", plus rec_63 (the
|
||||
recovery_mom input); the EDGAR fundamentals (research/lab/fundamentals.py;
|
||||
size = -log market cap); the alt-data panels (insider Form 4, Lazy Prices,
|
||||
earnings-announcement return, FINRA short-volume ratio lagged 2 sessions,
|
||||
SEC fails-to-deliver lagged 21 calendar days); and three house composites
|
||||
(mean member percentile ranks):
|
||||
house_overnight_recovery overnight_mom + rec_126 (overnight_recovery_top10)
|
||||
house_recovery_mom rec_63 + mom_12_1 (recovery_mom_top10)
|
||||
house_low_risk ivol + low_beta (lowrisk_top20)
|
||||
Every score is "higher = better" as the library defines it. A negative
|
||||
spread means the factor works the other way round.
|
||||
|
||||
Measures (members with a score and a forward return, >= 50 names a day):
|
||||
ic daily Spearman rank IC against the open(t+1) -> close(t+21) return
|
||||
ls top-quintile minus bottom-quintile 21-day return, annualised x 12
|
||||
top_x top quintile minus the member mean, annualised
|
||||
t Newey-West, 21 lags, on the daily overlapping series
|
||||
turn share of the top quintile replaced after 21 days
|
||||
top20 long-only top 20, every 21 days, buffer 2, next-open fills,
|
||||
10 bp a side, phase-averaged (us_sweep.run): excess Sharpe, CAGR,
|
||||
MaxDD, and alpha against the equal-weight member basket
|
||||
(Newey-West t), from the factor's coverage start
|
||||
corr correlation of the ls series with house_overnight_recovery's and
|
||||
mom_12_1's; span_t is the t of the intercept when the factor's
|
||||
ls is regressed on house_overnight_recovery's ls
|
||||
Periods: 2006-10..2016 and 2017..2026-09 (halves), 2014..2026-09 (after
|
||||
the survivor-biased years). A factor whose data start later uses what it
|
||||
has; a half with less than 2 years is left out.
|
||||
|
||||
Classes, fixed before the run:
|
||||
A works alone: ls t >= 2 over the span, ls > 0 in every usable half, and
|
||||
top-20 alpha vs EW > 0 in every usable half
|
||||
B long-short only: ls t >= 2 and > 0 in every half, but the long-only
|
||||
top 20 has no alpha in some half
|
||||
C unstable or weak: ls t >= 2 with a half <= 0, or 1 <= t < 2 with every
|
||||
half > 0
|
||||
R works reversed: ls t <= -2
|
||||
N nothing: the rest
|
||||
About 70 factors are tested at once, so roughly 3 will pass t >= 2 by
|
||||
chance. The t >= 3 bar (Harvey, Liu & Zhu 2016) is reported too.
|
||||
Descriptive census, no strategy gate. The top-20 runs go to
|
||||
trials_us.jsonl. The script refuses to overwrite census.csv.
|
||||
|
||||
--part construction (reported): the house inputs and the census's strongest
|
||||
long-only rows at the live overnight_recovery_top10 construction (top 10,
|
||||
every 10 days, buffer 2), vs the census construction. Writes construction.csv.
|
||||
|
||||
Result (2026-10-12), 86 factors:
|
||||
- Classes: 1 A, 2 B, 11 C, 3 R, 69 N. Six spreads reach |t| >= 2, about
|
||||
what chance gives at ~80 tests. No spread reaches |t| >= 3.
|
||||
- A low_svr21 (2018-26 only): IC t 3.0, spread +4.2%/yr (t 2.5), top-20
|
||||
alpha vs EW +5.5% (t 2.8), corr 0.27 with the house composite, span t 2.7.
|
||||
- B accruals: spread +2.8% (t 2.35), positive in both halves. Top-20 alpha
|
||||
+4.2% (t 1.75), but 2010-16 is -0.8%.
|
||||
- B rev_5: spread +4.3% (t 2.8), both halves. 77% of the top quintile turns
|
||||
over each month, and the net top-20 alpha is -0.7%.
|
||||
- R max_ret -6.9% (t -2.0): lottery names won. overnight_mom_1 and
|
||||
overnight_20 -5.2% / -4.5% (t -2.3 / -2.1): last month's overnight gains
|
||||
reverse.
|
||||
- Momentum and the house composites have no quintile spread (mom_12_1 -4.5%,
|
||||
t -1.05; 2006-16 -10%, 2017-26 +1.5%; overnight_mom -1.5%, t -0.56).
|
||||
Their edge is only in the extreme top names. Top 10 / every 10 / buffer 2
|
||||
vs EW:
|
||||
house_overnight_recovery +6.4% (t 1.40), halves -0.2 / +13.6
|
||||
overnight_mom alone +4.4% (t 0.97), halves +4.4 / +4.7
|
||||
rec_126 alone +2.0%
|
||||
mom_12_1 -0.4%
|
||||
At top 20 / every 21: +2.9 / +1.3 / -0.3 / -2.2.
|
||||
- Low-risk factors have negative raw spreads (low_vol -6.4%, low_beta -6.1%)
|
||||
because high beta won the bull market. On a beta-adjusted basis
|
||||
house_low_risk is +0.9% (t 0.57).
|
||||
- Long-only alpha with both halves positive (fundamentals start in 2010):
|
||||
roe +4.0% (t 2.75), net_issuance +3.1% (2.0), sp +3.0% (1.4; top 10
|
||||
+7.1%, t 2.5, found after looking). Their quintile spreads are flat.
|
||||
- CN volume factors invert in US large caps: pv_corr top-20 alpha -3.8%
|
||||
(t -3.1), turn_vol -3.5% (-2.9), vol_cv -3.4% (-2.8).
|
||||
- Boundary: S&P 500 only, with pre-2014 survivor bias. The fundamental and
|
||||
short-side data start later, so their 2006-16 half is shorter or missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pickle
|
||||
import time
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from research.lab import fundamentals as fu
|
||||
from research.lab.alpha import _nw_se, alpha_stats
|
||||
from research.lab.factor_scan import RESULTS, load
|
||||
from research.lab.factors import FACTORS
|
||||
from research.lab.levered_book import stats, tbill
|
||||
from research.lab.us_sweep import run
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
OUT = os.path.join(RESULTS, "factor_census")
|
||||
END = "2026-09-24"
|
||||
SPAN = ("2006-10-02", END)
|
||||
HALVES = {"h1": ("2006-10-02", "2016-12-30"), "h2": ("2017-01-03", END)}
|
||||
POST = ("2014-01-02", END)
|
||||
H = 21
|
||||
|
||||
FAMILY = {
|
||||
**dict.fromkeys(["mom_12_1", "mom_6_1", "mom_12_7", "mom_3", "resid_mom", "resid_mom_6", "sharpe_mom",
|
||||
"high_52w", "fip", "mom_fip", "rec_126", "rec_63", "ma200_dist", "ts_trend", "sector_mom",
|
||||
"trend_hzz", "amp_cut_mom", "stmom_hiturn", "seasonality"], "momentum/trend"),
|
||||
**dict.fromkeys(["overnight_mom", "overnight_mom_6", "overnight_mom_1", "overnight_20", "intraday_rev"],
|
||||
"overnight/intraday"),
|
||||
**dict.fromkeys(["rev_5", "rev_21", "rev_resid_21", "rev_sector_21"], "reversal"),
|
||||
**dict.fromkeys(["ivol", "ivol_20", "low_vol", "low_beta", "max_ret", "skew", "ideal_amp"], "low risk/lottery"),
|
||||
**dict.fromkeys(["amihud", "abn_volume", "low_turnover", "abn_turnover", "turn_vol", "pv_corr", "upvol_ratio",
|
||||
"abn_vol_ratio", "vol_cv"], "volume/liquidity"),
|
||||
**dict.fromkeys(["ep", "cfp", "sp", "bm", "fcfp", "shareholder_yield", "size"], "value/size"),
|
||||
**dict.fromkeys(["gp_a", "op_a", "cfo_a", "roe", "roa", "gross_margin", "fscore", "leverage", "accruals"],
|
||||
"quality/profitability"),
|
||||
**dict.fromkeys(["asset_growth", "net_issuance"], "investment/issuance"),
|
||||
**dict.fromkeys(["rev_growth", "gm_change", "roa_change", "sue", "sue_rev", "ear", "ear_z", "ear_count"],
|
||||
"growth/earnings"),
|
||||
}
|
||||
|
||||
|
||||
def nw_mean_t(x: pd.Series, lags: int = H) -> tuple[float, float, int]:
|
||||
v = x.dropna().to_numpy()
|
||||
n = len(v)
|
||||
if n < 250:
|
||||
return float("nan"), float("nan"), n
|
||||
e = v - v.mean()
|
||||
s = e @ e / n
|
||||
for k in range(1, lags + 1):
|
||||
s += 2 * (1 - k / (lags + 1)) * (e[k:] @ e[:-k]) / n
|
||||
return float(v.mean()), float(v.mean() / np.sqrt(s / n)), n
|
||||
|
||||
|
||||
def cross_section(score: pd.DataFrame, fwd: pd.DataFrame, member: pd.DataFrame):
|
||||
valid = member & fwd.notna() & score.notna()
|
||||
n = valid.sum(axis=1)
|
||||
S, F = score.where(valid), fwd.where(valid)
|
||||
q = S.rank(axis=1, pct=True)
|
||||
fr = F.rank(axis=1, pct=True)
|
||||
qa, fa = q.sub(q.mean(axis=1), axis=0), fr.sub(fr.mean(axis=1), axis=0)
|
||||
ic = (qa * fa).sum(axis=1) / np.sqrt((qa ** 2).sum(axis=1) * (fa ** 2).sum(axis=1))
|
||||
top, bot = q > 0.8, q <= 0.2
|
||||
ok = (n >= 50) & (top.sum(axis=1) >= 10) & (bot.sum(axis=1) >= 10)
|
||||
ls = (F.where(top).mean(axis=1) - F.where(bot).mean(axis=1)).where(ok)
|
||||
tx = (F.where(top).mean(axis=1) - F.mean(axis=1)).where(ok)
|
||||
stay = (top & top.shift(-H, fill_value=False)).sum(axis=1) / top.sum(axis=1).replace(0, np.nan)
|
||||
return ic.where(ok), ls, tx, (1 - stay).where(ok), n
|
||||
|
||||
|
||||
def span_t(y: pd.Series, x: pd.Series) -> float:
|
||||
d = pd.concat([y, x], axis=1).dropna()
|
||||
if len(d) < 250:
|
||||
return float("nan")
|
||||
X = np.c_[np.ones(len(d)), d.iloc[:, 1].to_numpy()]
|
||||
b, *_ = np.linalg.lstsq(X, d.iloc[:, 0].to_numpy(), rcond=None)
|
||||
e = d.iloc[:, 0].to_numpy() - X @ b
|
||||
return float(b[0] / _nw_se(X, e, H)[0])
|
||||
|
||||
|
||||
def pct_rank(df: pd.DataFrame, member: pd.DataFrame) -> pd.DataFrame:
|
||||
return df.where(member).rank(axis=1, pct=True)
|
||||
|
||||
|
||||
def scores(p) -> dict[str, pd.DataFrame]:
|
||||
cols, dates = p.close.columns, p.dates
|
||||
S = {}
|
||||
for n, f in FACTORS.items():
|
||||
if "us" in f.markets:
|
||||
S[n] = f(p).reindex(index=dates, columns=cols)
|
||||
c = p.close.ffill()
|
||||
S["rec_63"] = c / c.rolling(63, min_periods=63).min() - 1
|
||||
S["house_overnight_recovery"] = (pct_rank(S["overnight_mom"], p.member) + pct_rank(S["rec_126"], p.member)) / 2
|
||||
S["house_recovery_mom"] = (pct_rank(S["rec_63"], p.member) + pct_rank(S["mom_12_1"], p.member)) / 2
|
||||
S["house_low_risk"] = (pct_rank(S["ivol"], p.member) + pct_rank(S["low_beta"], p.member)) / 2
|
||||
F = fu.load(p)
|
||||
for k, v in F.items():
|
||||
v = v.reindex(index=dates, columns=cols).astype(float)
|
||||
if k == "mcap":
|
||||
S["size"] = -np.log(v)
|
||||
else:
|
||||
S[k] = v
|
||||
for path, keep in (("insider_factors.pkl", None), ("lazy_prices_factors.pkl", None),
|
||||
("earnings_factors.pkl", ["ear", "ear_z", "ear_count"])):
|
||||
with open(os.path.join("data", "research", path), "rb") as fh:
|
||||
d = pickle.load(fh)
|
||||
for k, v in d.items():
|
||||
if keep is None or k in keep:
|
||||
S[k] = v.reindex(index=dates, columns=cols).astype(float)
|
||||
from research.lab.short_volume import factors as svr_factors, load_svr
|
||||
for k, v in svr_factors(load_svr(p.close)).items():
|
||||
if k in ("svr21", "svr_z"):
|
||||
S[f"low_{k}"] = -v.shift(2).reindex(index=dates, columns=cols)
|
||||
from research.lab.fails import PUBLISH_LAG_DAYS, load_ftd
|
||||
ftd = load_ftd(p.close, p.volume)["ftd_int"]
|
||||
ftd.index = ftd.index + pd.Timedelta(days=PUBLISH_LAG_DAYS)
|
||||
S["low_ftd_int"] = -(ftd.reindex(index=dates.union(ftd.index), columns=cols).ffill(limit=45).reindex(dates))
|
||||
for k in list(S):
|
||||
if k.startswith(("insider_", "lazy_")):
|
||||
FAMILY.setdefault(k, "alt: insider" if k.startswith("insider_") else "alt: text (Lazy Prices)")
|
||||
elif k.startswith("low_svr") or k == "low_ftd_int":
|
||||
FAMILY.setdefault(k, "alt: short side")
|
||||
elif k.startswith("house_"):
|
||||
FAMILY.setdefault(k, "house composite")
|
||||
return S
|
||||
|
||||
|
||||
def classify(r: dict) -> str:
|
||||
halves = [r[f"ls_{h}"] for h in HALVES if np.isfinite(r.get(f"ls_{h}", np.nan))]
|
||||
alphas = [r[f"alpha_{h}"] for h in HALVES if np.isfinite(r.get(f"alpha_{h}", np.nan))]
|
||||
t = r["ls_t"]
|
||||
if not np.isfinite(t):
|
||||
return "N"
|
||||
if t <= -2:
|
||||
return "R"
|
||||
if t >= 2 and halves and all(h > 0 for h in halves):
|
||||
return "A" if alphas and all(a > 0 for a in alphas) else "B"
|
||||
if t >= 2 or (t >= 1 and halves and all(h > 0 for h in halves)):
|
||||
return "C"
|
||||
return "N"
|
||||
|
||||
|
||||
CONSTRUCTION = ["house_overnight_recovery", "overnight_mom", "rec_126", "house_recovery_mom", "rec_63", "mom_12_1",
|
||||
"house_low_risk", "low_svr21", "roe", "net_issuance", "accruals", "sp"]
|
||||
|
||||
|
||||
def construction(p, S) -> None:
|
||||
out_csv = os.path.join(OUT, "construction.csv")
|
||||
if os.path.exists(out_csv):
|
||||
raise SystemExit(f"{out_csv} exists; this census does not overwrite results")
|
||||
ret = p.close.pct_change(fill_method=None)
|
||||
ew = ret.where(p.member.shift(1, fill_value=False)).mean(axis=1)
|
||||
rf = tbill(p.dates).fillna(0.0)
|
||||
sp = p.slice("2006-06-01", END)
|
||||
rows = []
|
||||
for name in CONSTRUCTION:
|
||||
sc = S[name].where(p.member)
|
||||
n = sc.notna().sum(axis=1)
|
||||
a = max(pd.Timestamp(SPAN[0]), n[n >= 100].index[0] + pd.Timedelta(days=31))
|
||||
for label, kw in (("top10_e10_b2", dict(top_n=10, every=10, buffer=2.0)),
|
||||
("top20_e21_b2", dict(top_n=20, every=H, buffer=2.0))):
|
||||
rr = run(sp, sc, [(a.date().isoformat(), END)], **kw)["_rets"].loc[a:END]
|
||||
s, al = stats(rr, rf), alpha_stats(rr, ew.reindex(rr.index), rf.reindex(rr.index))
|
||||
r = {"factor": name, "construction": label, "start": a.date().isoformat(), "cagr": float(s["cagr"]),
|
||||
"sharpe": float(s["sharpe"]), "mdd": float(s["mdd"]), "alpha": float(al["alpha"]),
|
||||
"alpha_t": float(al["t"]), "beta": float(al["beta"])}
|
||||
for h, (ha, hb) in HALVES.items():
|
||||
seg = rr.loc[ha:hb]
|
||||
r[f"alpha_{h}"] = (float(alpha_stats(seg, ew.reindex(seg.index), rf.reindex(seg.index))["alpha"])
|
||||
if len(seg) >= 2 * 252 else float("nan"))
|
||||
rows.append(r)
|
||||
print(f"{name:26s} {label:13s} cagr {r['cagr']*100:5.1f}% sh {r['sharpe']:.2f} mdd {r['mdd']*100:6.1f}% "
|
||||
f"alpha {r['alpha']*100:+5.1f}% (t {r['alpha_t']:+.2f}) beta {r['beta']:.2f} halves "
|
||||
f"{r['alpha_h1']*100:+5.1f} / {r['alpha_h2']*100:+5.1f}", flush=True)
|
||||
pd.DataFrame(rows).to_csv(out_csv, index=False)
|
||||
from research.lab.trials import append_trials
|
||||
append_trials(os.path.join(RESULTS, "trials_us.jsonl"),
|
||||
[{"name": f"factor_census:{r['factor']}:{r['construction']}", **{k: r[k] for k in
|
||||
("cagr", "sharpe", "mdd", "alpha", "alpha_t")}} for r in rows],
|
||||
{"study": "factor_census construction", "role": "reported; previously inspected history"})
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--part", choices=["census", "construction"], default="census")
|
||||
part = ap.parse_args().part
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
if part == "construction":
|
||||
p, _ = load("us")
|
||||
construction(p, scores(p))
|
||||
return
|
||||
out_csv = os.path.join(OUT, "census.csv")
|
||||
if os.path.exists(out_csv):
|
||||
raise SystemExit(f"{out_csv} exists; this census does not overwrite results")
|
||||
p, _ = load("us")
|
||||
t0 = time.time()
|
||||
S = scores(p)
|
||||
print(f"{len(S)} factors built [{time.time() - t0:.0f}s]", flush=True)
|
||||
o = p.open.where(p.open > 0)
|
||||
fwd = p.close.ffill().shift(-H) / o.shift(-1) - 1 # a delisted name keeps its last print
|
||||
member = p.member
|
||||
ret = p.close.pct_change(fill_method=None)
|
||||
ew = ret.where(member.shift(1, fill_value=False)).mean(axis=1)
|
||||
rf = tbill(p.dates).fillna(0.0)
|
||||
sp = p.slice("2006-06-01", END)
|
||||
|
||||
LS, rows = {}, []
|
||||
for name, sc in S.items():
|
||||
t = time.time()
|
||||
ic, ls, tx, turn, n = cross_section(sc, fwd, member)
|
||||
ic, ls, tx, turn, n = (x.loc[SPAN[0]:SPAN[1]] for x in (ic, ls, tx, turn, n))
|
||||
LS[name] = ls
|
||||
cov = n[n >= 100]
|
||||
start = cov.index[0] if len(cov) else None
|
||||
r = {"factor": name, "family": FAMILY.get(name, "other"),
|
||||
"start": start.date().isoformat() if start is not None else None,
|
||||
"names_2025": float(n.loc["2025"].mean()) if len(n.loc["2025"]) else float("nan")}
|
||||
r["ic"], r["ic_t"], _ = nw_mean_t(ic)
|
||||
m, r["ls_t"], _ = nw_mean_t(ls)
|
||||
r["ls"] = m * 12
|
||||
m, r["top_x_t"], _ = nw_mean_t(tx)
|
||||
r["top_x"] = m * 12
|
||||
r["turn"] = float(turn.mean())
|
||||
for h, (a, b) in {**HALVES, "post2014": POST}.items():
|
||||
m, tt, nn = nw_mean_t(ls.loc[a:b])
|
||||
ok = nn >= 2 * 252
|
||||
r[f"ls_{h}"], r[f"ls_t_{h}"] = (m * 12, tt) if ok else (float("nan"), float("nan"))
|
||||
if start is not None:
|
||||
a = max(pd.Timestamp(SPAN[0]), start + pd.Timedelta(days=31))
|
||||
st = run(sp, sc.where(member), [(a.date().isoformat(), END)], top_n=20, buffer=2.0, every=H)
|
||||
rr = st["_rets"].loc[a:END]
|
||||
s = stats(rr, rf)
|
||||
al = alpha_stats(rr, ew.reindex(rr.index), rf.reindex(rr.index))
|
||||
r.update({"top20_sharpe": float(s["sharpe"]), "top20_cagr": float(s["cagr"]), "top20_mdd": float(s["mdd"]),
|
||||
"alpha": float(al["alpha"]), "alpha_t": float(al["t"]), "top20_turn": float(st["turn"])})
|
||||
for h, (ha, hb) in HALVES.items():
|
||||
seg = rr.loc[ha:hb]
|
||||
if len(seg) >= 2 * 252:
|
||||
r[f"alpha_{h}"] = float(alpha_stats(seg, ew.reindex(seg.index), rf.reindex(seg.index))["alpha"])
|
||||
rows.append(r)
|
||||
print(f"{name:26s} ic {r['ic']:+.3f} (t {r['ic_t']:+.1f}) ls {r['ls']*100:+6.1f}% t {r['ls_t']:+5.2f} "
|
||||
f"[{r.get('ls_h1', np.nan)*100:+5.1f} / {r.get('ls_h2', np.nan)*100:+5.1f}] "
|
||||
f"top20 sh {r.get('top20_sharpe', np.nan):.2f} alpha {r.get('alpha', np.nan)*100:+5.1f}% "
|
||||
f"(t {r.get('alpha_t', np.nan):+.1f}) [{time.time() - t:.0f}s]", flush=True)
|
||||
|
||||
L = pd.DataFrame(LS)
|
||||
for r in rows:
|
||||
y = L[r["factor"]]
|
||||
r["corr_house"] = float(y.corr(L["house_overnight_recovery"]))
|
||||
r["corr_mom"] = float(y.corr(L["mom_12_1"]))
|
||||
r["span_t_house"] = span_t(y, L["house_overnight_recovery"])
|
||||
r["class"] = classify(r)
|
||||
r["t3"] = bool(abs(r["ls_t"]) >= 3) if np.isfinite(r["ls_t"]) else False
|
||||
df = pd.DataFrame(rows)
|
||||
df.to_csv(out_csv, index=False)
|
||||
L.to_pickle(os.path.join(OUT, "ls_returns.pkl"))
|
||||
with pd.option_context("display.width", 250, "display.max_rows", 200):
|
||||
show = df.sort_values(["family", "ls_t"], ascending=[True, False])
|
||||
cols = ["family", "factor", "class", "start", "ic", "ic_t", "ls", "ls_t", "ls_h1", "ls_h2", "ls_t_post2014",
|
||||
"top20_sharpe", "alpha", "alpha_t", "alpha_h1", "alpha_h2", "corr_house", "span_t_house", "turn"]
|
||||
print(show[cols].round(3).to_string(index=False))
|
||||
from research.lab.trials import append_trials
|
||||
append_trials(os.path.join(RESULTS, "trials_us.jsonl"),
|
||||
[{"name": f"factor_census:{r['factor']}:top20", **{k: r[k] for k in
|
||||
("top20_sharpe", "top20_cagr", "top20_mdd", "alpha", "alpha_t") if k in r}} for r in rows],
|
||||
{"study": "factor_census", "role": "descriptive census; previously inspected history"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
676
research/lab/fetch_options.py
Normal file
676
research/lab/fetch_options.py
Normal file
@@ -0,0 +1,676 @@
|
||||
"""
|
||||
Option data for the S&P 500, for options-implied stock factors:
|
||||
- IV skew (Xing, Zhang & Zhao 2010)
|
||||
- the call-put IV spread (Cremers & Weinbaum 2010)
|
||||
- option/stock volume (Johnson & So 2012)
|
||||
- implied-vol level and changes
|
||||
|
||||
uv run python -m research.lab.fetch_options snapshot # after the close (systemd timer)
|
||||
uv run python -m research.lab.fetch_options snapshot --limit 5 --force --out /tmp/x # smoke test
|
||||
uv run python -m research.lab.fetch_options backfill # OPRA daily bars since 2024-02 (resumable)
|
||||
uv run python -m research.lab.fetch_options volume-panel # daily option volume per underlying
|
||||
uv run python -m research.lab.fetch_options status # runs, backfill progress, quote-quality verdict
|
||||
|
||||
Sources: Alpaca. Keys are ALPACA_API_KEY / ALPACA_API_SECRET from .env and
|
||||
are never logged. The account has the free Basic plan.
|
||||
- Option quotes come from /v1beta1/options/snapshots, feed=indicative,
|
||||
with Alpaca's IV and greeks. Alpaca's docs say indicative quotes "are not
|
||||
actual OPRA quotes". The OPRA BBO needs Algo Trader Plus ($99/month)
|
||||
plus a signed OPRA agreement. Quotes have no free history, so they are
|
||||
collected forward from 2026-10-12.
|
||||
- Option bars come from /v1beta1/options/bars. They are computed from
|
||||
OPRA trades, so they are real prints. History starts in 2024-02, and
|
||||
Basic cannot read the last 15 minutes. Volume factors use these bars,
|
||||
backfilled.
|
||||
- Open interest comes from /v2/options/contracts (OCC, prior day); a
|
||||
null means none was reported. Stock prices are SIP daily bars,
|
||||
unadjusted, so they match the strikes.
|
||||
|
||||
snapshot (16:25 America/New_York, Mon-Fri, deploy/systemd/quant-trading-options.timer)
|
||||
The Alpaca calendar decides the session: the latest one that closed at
|
||||
least 10 minutes ago. The run is skipped if the next session has opened
|
||||
or the session's chain file exists, unless --force.
|
||||
universe data/universe_sp500.json + SPY, QQQ, IWM
|
||||
quotes expiries 5-100 days, strikes 0.7-1.3 x close
|
||||
bars + OI every contract expiring within 0-100 days, all strikes
|
||||
quality for each underlying, the ATM call, the ATM put and the
|
||||
0.95-moneyness put of the expiry nearest 30 days (14-45 days)
|
||||
are compared with OPRA prints. The print is the last 1-minute
|
||||
bar from 15:45-16:00, and the reference is the indicative bid,
|
||||
ask and mid. Measured: share of prints inside the quote,
|
||||
position in the spread, IV(print) - IV(mid).
|
||||
Pre-set rule: once >= 20 sessions are in, indicative quotes are
|
||||
adequate if >= 90% of prints are inside the quote and the
|
||||
median |IV(print) - IV(mid)| <= 1 vol point. Otherwise
|
||||
consider the OPRA plan. `status` prints the verdict.
|
||||
backfill
|
||||
- Universe: every name that was in the S&P 500 at some point since
|
||||
2024-02-01 (data/sp500_history.json), plus the three ETFs, so names that
|
||||
left the index are kept.
|
||||
- Contracts: expired and active, expiring from 2024-02-01 to end+100 days.
|
||||
- Bars: daily, from 100 days before each expiry to its expiry, for every
|
||||
strike.
|
||||
- One file per underlying, so the run resumes. It pauses 16:10-16:55 New
|
||||
York on weekdays so it does not compete with snapshot for the
|
||||
200/minute limit.
|
||||
|
||||
Files (data/options/us/):
|
||||
chains/YYYY/chain_<session>.csv.gz quotes, OPRA bar, OI, Alpaca IV/greeks, close,
|
||||
r, implied forward, own IVs (iv_fwd, iv_spot)
|
||||
summary/summary_<session>.csv per underlying: iv_atm30, skew_xzz, cp_spread,
|
||||
option volume (OPRA), os_volume, pc ratios
|
||||
bars/YYYY/bars_<session>.csv.gz OPRA daily bars, 0-100 day expiries
|
||||
bars_hist/<UNDERLYING>.csv.gz the backfill, same columns
|
||||
stock/stock_<session>.csv, stock_raw.csv.gz unadjusted stock close/volume
|
||||
quality/quality_<session>.csv the print-vs-quote sample
|
||||
volume_panel.csv.gz date x underlying option volume, built from bars
|
||||
manifest.jsonl, backfill_manifest.jsonl
|
||||
IVs are European. iv_fwd is Black-76 on the forward implied by put-call
|
||||
parity at the three strikes nearest the close, so dividends drop out.
|
||||
iv_spot is Black-Scholes on the close with no dividends; cp_spread uses it
|
||||
(as Cremers-Weinbaum do), so dividend payers read negative. American early
|
||||
exercise is ignored, so the summaries use near-the-money strikes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import requests
|
||||
from scipy.special import ndtr
|
||||
|
||||
from research.lab.fetch_alpaca import _keys
|
||||
|
||||
NY = ZoneInfo("America/New_York")
|
||||
DATA = "https://data.alpaca.markets"
|
||||
TRADE = "https://paper-api.alpaca.markets"
|
||||
OUT = os.path.join("data", "options", "us")
|
||||
UNIVERSE = os.path.join("data", "universe_sp500.json")
|
||||
HISTORY = os.path.join("data", "sp500_history.json")
|
||||
RF_PATH = os.path.join("data", "research", "ext", "fred_dtb3.csv")
|
||||
EXTRA = ["SPY", "QQQ", "IWM"]
|
||||
DTE_MIN, DTE_MAX = 5, 100
|
||||
MNY_LO, MNY_HI = 0.7, 1.3
|
||||
HIST_START = "2024-02-01"
|
||||
QUALITY_MIN_SESSIONS, QUALITY_INSIDE, QUALITY_IV = 20, 0.90, 0.01
|
||||
OCC = re.compile(r"^[A-Z]{1,5}\d{6,7}[CP]\d{8}$") # what the bars endpoint accepts; adjusted roots like 1AAPL fail
|
||||
BAR_COLS = ["date", "underlying", "symbol", "type", "expiry", "strike", "o", "h", "l", "c", "v", "n", "vw"]
|
||||
|
||||
|
||||
class Api:
|
||||
"""Paced GET with retries. ``yield_to_snapshot`` sleeps through 16:10-16:55
|
||||
New York on weekdays so a backfill leaves the rate limit to the daily run."""
|
||||
|
||||
def __init__(self, min_interval: float = 0.32, yield_to_snapshot: bool = False) -> None:
|
||||
self.s = requests.Session()
|
||||
self.s.headers.update(_keys())
|
||||
self.min_interval = min_interval
|
||||
self.yield_to_snapshot = yield_to_snapshot
|
||||
self.next = 0.0
|
||||
self.n = 0
|
||||
|
||||
def _quiet_window(self) -> None:
|
||||
now = dt.datetime.now(NY)
|
||||
start = now.replace(hour=16, minute=10, second=0, microsecond=0)
|
||||
end = now.replace(hour=16, minute=55, second=0, microsecond=0)
|
||||
if now.weekday() < 5 and start <= now < end:
|
||||
print(f" pausing until 16:55 New York for the daily snapshot", flush=True)
|
||||
time.sleep((end - now).total_seconds())
|
||||
|
||||
def get(self, url: str, params: dict) -> dict:
|
||||
for attempt in range(7):
|
||||
if self.yield_to_snapshot:
|
||||
self._quiet_window()
|
||||
wait = self.next - time.time()
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
self.next = time.time() + self.min_interval
|
||||
try:
|
||||
r = self.s.get(url, params=params, timeout=90)
|
||||
except requests.RequestException:
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
self.n += 1
|
||||
if r.status_code == 429 or r.status_code >= 500:
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"HTTP {r.status_code} {r.text[:200]}")
|
||||
return r.json()
|
||||
raise RuntimeError("retries exhausted")
|
||||
|
||||
def pages(self, url: str, params: dict, key: str):
|
||||
params = dict(params)
|
||||
while True:
|
||||
j = self.get(url, params)
|
||||
yield j.get(key) or ({} if key in ("snapshots", "bars") else [])
|
||||
tok = j.get("next_page_token")
|
||||
if not tok:
|
||||
return
|
||||
params["page_token"] = tok
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# inputs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def pick_session(api: Api, now: dt.datetime, force: bool) -> dt.date | None:
|
||||
today = now.astimezone(NY).date()
|
||||
cal = api.get(f"{TRADE}/v2/calendar", {"start": (today - dt.timedelta(days=10)).isoformat(),
|
||||
"end": (today + dt.timedelta(days=10)).isoformat()})
|
||||
days = []
|
||||
for c in cal:
|
||||
d = dt.date.fromisoformat(c["date"])
|
||||
at = lambda hm: dt.datetime.combine(d, dt.time(int(hm[:2]), int(hm[3:5])), NY) # noqa: E731
|
||||
days.append((d, at(c["open"]), at(c["close"])))
|
||||
done = [x for x in days if x[2] + dt.timedelta(minutes=10) <= now]
|
||||
if not done:
|
||||
print("skip: no session in the calendar window has closed")
|
||||
return None
|
||||
d, _, _ = done[-1]
|
||||
nxt = [x for x in days if x[0] > d]
|
||||
if nxt and now >= nxt[0][1] and not force:
|
||||
print(f"skip: session {d} is over and {nxt[0][0]} has opened; quotes would be from another day")
|
||||
return None
|
||||
return d
|
||||
|
||||
|
||||
def universe() -> list[str]:
|
||||
with open(UNIVERSE) as fh:
|
||||
ticks = json.load(fh)["tickers"]
|
||||
return sorted({t.replace("-", ".") for t in ticks} | set(EXTRA))
|
||||
|
||||
|
||||
def pit_universe(start: str, end: str) -> list[str]:
|
||||
"""Every name in the S&P 500 at some point in [start, end], plus EXTRA."""
|
||||
with open(HISTORY) as fh:
|
||||
iv = json.load(fh)["intervals"]
|
||||
keep = {t for t, spans in iv.items()
|
||||
if any((a or "0000") <= end and (b is None or b >= start) for a, b in spans)}
|
||||
return sorted({t.replace("-", ".") for t in keep} | set(universe()))
|
||||
|
||||
|
||||
def stock_bars(api: Api, syms: list[str], start: str, end: dt.datetime) -> pd.DataFrame:
|
||||
rows = []
|
||||
for i in range(0, len(syms), 100):
|
||||
params = {"symbols": ",".join(syms[i:i + 100]), "timeframe": "1Day", "start": start,
|
||||
"end": end.isoformat(), "feed": "sip", "adjustment": "raw", "limit": 10000}
|
||||
for bars in api.pages(f"{DATA}/v2/stocks/bars", params, "bars"):
|
||||
for sym, bl in bars.items():
|
||||
rows += [{"date": b["t"][:10], "symbol": sym, "close": b["c"], "volume": b["v"]} for b in bl]
|
||||
return pd.DataFrame(rows, columns=["date", "symbol", "close", "volume"])
|
||||
|
||||
|
||||
def list_contracts(api: Api, syms: list[str], gte: str, lte: str, status: str = "active") -> list[dict]:
|
||||
out = []
|
||||
for i in range(0, len(syms), 10):
|
||||
params = {"underlying_symbols": ",".join(syms[i:i + 10]), "status": status,
|
||||
"expiration_date_gte": gte, "expiration_date_lte": lte, "limit": 10000}
|
||||
try:
|
||||
for page in api.pages(f"{TRADE}/v2/options/contracts", params, "option_contracts"):
|
||||
for c in page:
|
||||
out.append({"symbol": c["symbol"], "underlying": c["underlying_symbol"],
|
||||
"type": "C" if c["type"] == "call" else "P", "expiry": c["expiration_date"],
|
||||
"strike": float(c["strike_price"]),
|
||||
"oi": float(c["open_interest"]) if c.get("open_interest") else np.nan,
|
||||
"oi_date": c.get("open_interest_date")})
|
||||
except RuntimeError as exc:
|
||||
print(f"contracts {syms[i]}..: {exc}", flush=True)
|
||||
return out
|
||||
|
||||
|
||||
def option_bars(api: Api, symbols: list[str], start: str, end: str, timeframe: str = "1Day") -> list[dict]:
|
||||
"""OPRA-derived bars. Contracts adjusted for corporate actions (non-standard
|
||||
roots) are skipped: the endpoint rejects their symbols."""
|
||||
symbols = [x for x in symbols if OCC.match(x)]
|
||||
# Basic cannot read the last 15 minutes, and a date-only end of today counts
|
||||
# as all of today (HTTP 403 "OPRA agreement is not signed"): send a timestamp
|
||||
# no later than 16 minutes ago.
|
||||
e = (dt.datetime.combine(dt.date.fromisoformat(end), dt.time(0), NY) if len(end) == 10
|
||||
else dt.datetime.fromisoformat(end))
|
||||
end = min(e, dt.datetime.now(NY) - dt.timedelta(minutes=16)).isoformat()
|
||||
rows = []
|
||||
for i in range(0, len(symbols), 1000):
|
||||
params = {"symbols": ",".join(symbols[i:i + 1000]), "timeframe": timeframe, "start": start, "end": end,
|
||||
"limit": 10000}
|
||||
for bars in api.pages(f"{DATA}/v1beta1/options/bars", params, "bars"):
|
||||
for sym, bl in bars.items():
|
||||
rows += [{"symbol": sym, "t": b["t"], "o": b["o"], "h": b["h"], "l": b["l"], "c": b["c"],
|
||||
"v": b["v"], "n": b.get("n"), "vw": b.get("vw")} for b in bl]
|
||||
return rows
|
||||
|
||||
|
||||
def chain(api: Api, sym: str, close: float, session: dt.date) -> list[dict]:
|
||||
params = {"feed": "indicative", "limit": 1000,
|
||||
"expiration_date_gte": (session + dt.timedelta(days=DTE_MIN)).isoformat(),
|
||||
"expiration_date_lte": (session + dt.timedelta(days=DTE_MAX)).isoformat(),
|
||||
"strike_price_gte": round(MNY_LO * close, 2), "strike_price_lte": round(MNY_HI * close, 2)}
|
||||
rows = []
|
||||
for snaps in api.pages(f"{DATA}/v1beta1/options/snapshots/{sym}", params, "snapshots"):
|
||||
for occ, s in snaps.items():
|
||||
q, t, b, g = s.get("latestQuote") or {}, s.get("latestTrade") or {}, s.get("dailyBar") or {}, s.get("greeks") or {}
|
||||
today = b.get("t", "")[:10] == session.isoformat()
|
||||
rows.append({"underlying": sym, "symbol": occ, "type": occ[-9],
|
||||
"expiry": f"20{occ[-15:-13]}-{occ[-13:-11]}-{occ[-11:-9]}",
|
||||
"strike": int(occ[-8:]) / 1000, "bid": q.get("bp"), "ask": q.get("ap"),
|
||||
"bid_size": q.get("bs"), "ask_size": q.get("as"), "quote_time": q.get("t"),
|
||||
"ind_last": t.get("p"), "ind_last_time": t.get("t"),
|
||||
"ind_day_volume": b.get("v", 0) if today else 0,
|
||||
"alp_iv": s.get("impliedVolatility"), "delta": g.get("delta"), "gamma": g.get("gamma"),
|
||||
"theta": g.get("theta"), "vega": g.get("vega")})
|
||||
return rows
|
||||
|
||||
|
||||
def rate(session: dt.date) -> tuple[float, str]:
|
||||
try:
|
||||
rf = pd.read_csv(RF_PATH, index_col=0, parse_dates=True).iloc[:, 0]
|
||||
rf = pd.to_numeric(rf, errors="coerce").dropna().loc[:pd.Timestamp(session)]
|
||||
return float(rf.iloc[-1]) / 100, rf.index[-1].date().isoformat()
|
||||
except (OSError, IndexError):
|
||||
return 0.04, "default"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# implied volatility and per-underlying summary
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def black76(F, K, T, sig, call, disc):
|
||||
sd = sig * np.sqrt(T)
|
||||
d1 = (np.log(F / K) + 0.5 * sd ** 2) / sd
|
||||
d2 = d1 - sd
|
||||
return np.where(call, disc * (F * ndtr(d1) - K * ndtr(d2)), disc * (K * ndtr(-d2) - F * ndtr(-d1)))
|
||||
|
||||
|
||||
def implied_vol(price, F, K, T, call, disc, iters: int = 80) -> np.ndarray:
|
||||
price, F, K, T, call, disc = (np.asarray(x, dtype=float if i != 4 else bool)
|
||||
for i, x in enumerate((price, F, K, T, call, disc)))
|
||||
lo, hi = np.full(len(price), 1e-4), np.full(len(price), 5.0)
|
||||
intrinsic = disc * np.where(call, np.maximum(F - K, 0), np.maximum(K - F, 0))
|
||||
ok = np.isfinite(price) & (price > intrinsic + 1e-6) & (price < black76(F, K, T, hi, call, disc))
|
||||
for _ in range(iters):
|
||||
mid = (lo + hi) / 2
|
||||
above = black76(F, K, T, mid, call, disc) > price
|
||||
hi, lo = np.where(above, mid, hi), np.where(above, lo, mid)
|
||||
return np.where(ok, (lo + hi) / 2, np.nan)
|
||||
|
||||
|
||||
def add_iv(df: pd.DataFrame, r: float) -> pd.DataFrame:
|
||||
df["mid"] = (df["bid"] + df["ask"]) / 2
|
||||
good = (df["bid"] > 0) & (df["ask"] > df["bid"])
|
||||
df.loc[~good, "mid"] = np.nan
|
||||
df["T"] = df["dte"] / 365.0
|
||||
disc = np.exp(-r * df["T"])
|
||||
df["fwd"] = np.nan
|
||||
for _, g in df.groupby(["underlying", "expiry"]):
|
||||
piv = g.pivot_table(index="strike", columns="type", values="mid")
|
||||
if {"C", "P"} <= set(piv.columns):
|
||||
piv = piv.dropna()
|
||||
if len(piv):
|
||||
S = g["stock_close"].iloc[0]
|
||||
near = piv.iloc[np.argsort(np.abs(piv.index.to_numpy() - S))[:3]]
|
||||
f = near.index.to_numpy() + np.exp(r * g["T"].iloc[0]) * (near["C"] - near["P"]).to_numpy()
|
||||
df.loc[g.index, "fwd"] = float(np.median(f))
|
||||
nofwd = df["fwd"].isna()
|
||||
df.loc[nofwd, "fwd"] = df.loc[nofwd, "stock_close"] * np.exp(r * df.loc[nofwd, "T"])
|
||||
call = (df["type"] == "C").to_numpy()
|
||||
args = (df["strike"].to_numpy(), df["T"].to_numpy(), call, disc.to_numpy())
|
||||
df["iv_fwd"] = implied_vol(df["mid"].to_numpy(), df["fwd"].to_numpy(), *args)
|
||||
df["iv_spot"] = implied_vol(df["mid"].to_numpy(), (df["stock_close"] * np.exp(r * df["T"])).to_numpy(), *args)
|
||||
return df
|
||||
|
||||
|
||||
def summarize(g: pd.DataFrame, vol: dict) -> dict:
|
||||
S = g["stock_close"].iloc[0]
|
||||
v = g[g["iv_fwd"].notna()]
|
||||
out = {"underlying": g["underlying"].iloc[0], "stock_close": S, "stock_volume": g["stock_volume"].iloc[0],
|
||||
"n_contracts": len(g), "n_valid": len(v),
|
||||
"med_rel_spread": float(((v["ask"] - v["bid"]) / v["mid"]).median()) if len(v) else np.nan}
|
||||
atm = {}
|
||||
for _, x in v.groupby("expiry"):
|
||||
F = x["fwd"].iloc[0]
|
||||
k = x["strike"].iloc[np.argmin(np.abs(x["strike"].to_numpy() - F))]
|
||||
atm[int(x["dte"].iloc[0])] = float(x.loc[x["strike"] == k, "iv_fwd"].mean())
|
||||
ds = sorted(d for d in atm if 7 <= d <= DTE_MAX)
|
||||
if ds:
|
||||
lo = [d for d in ds if d <= 30]
|
||||
hi = [d for d in ds if d >= 30]
|
||||
if lo and hi and lo[-1] != hi[0]: # total variance, linear in time
|
||||
a, b = lo[-1], hi[0]
|
||||
w = atm[a] ** 2 * a + (atm[b] ** 2 * b - atm[a] ** 2 * a) * (30 - a) / (b - a)
|
||||
out["iv_atm30"] = float(np.sqrt(w / 30)) if w > 0 else np.nan
|
||||
else: # 30 days exactly, or only one side: nearest
|
||||
out["iv_atm30"] = atm[hi[0]] if hi else atm[lo[-1]]
|
||||
mid_exp = v[(v["dte"] >= 10) & (v["dte"] <= 60)]
|
||||
if len(mid_exp):
|
||||
e = mid_exp.loc[(mid_exp["dte"] - 30).abs().idxmin(), "expiry"]
|
||||
x = mid_exp[mid_exp["expiry"] == e]
|
||||
m = x["strike"] / S
|
||||
puts = x[(x["type"] == "P") & (m >= 0.80) & (m <= 0.95)]
|
||||
calls = x[(x["type"] == "C") & (m >= 0.95) & (m <= 1.05)]
|
||||
if len(puts) and len(calls):
|
||||
p = puts.loc[(puts["strike"] / S - 0.95).abs().idxmin(), "iv_fwd"]
|
||||
c = calls.loc[(calls["strike"] / S - 1.0).abs().idxmin(), "iv_fwd"]
|
||||
out["skew_xzz"] = float(p - c)
|
||||
pairs = mid_exp[(mid_exp["strike"] / S).between(0.8, 1.2) & mid_exp["iv_spot"].notna()]
|
||||
pv = pairs.pivot_table(index=["expiry", "strike"], columns="type", values=["iv_spot", "oi"])
|
||||
if ("iv_spot", "C") in pv.columns and ("iv_spot", "P") in pv.columns:
|
||||
pv = pv.dropna(subset=[("iv_spot", "C"), ("iv_spot", "P")])
|
||||
if len(pv):
|
||||
w = pv[("oi", "C")].fillna(0) + pv[("oi", "P")].fillna(0) if ("oi", "C") in pv and ("oi", "P") in pv \
|
||||
else pd.Series(1.0, index=pv.index)
|
||||
w = w if w.sum() > 0 else pd.Series(1.0, index=pv.index)
|
||||
out["cp_spread"] = float(((pv[("iv_spot", "C")] - pv[("iv_spot", "P")]) * w).sum() / w.sum())
|
||||
cv, pv_ = vol.get("C", np.nan), vol.get("P", np.nan)
|
||||
co, po = g.loc[g["type"] == "C", "oi"].sum(), g.loc[g["type"] == "P", "oi"].sum()
|
||||
out.update({"call_volume": cv, "put_volume": pv_, "call_oi": co, "put_oi": po,
|
||||
"pc_volume": pv_ / cv if cv else np.nan, "pc_oi": po / co if co else np.nan,
|
||||
"os_volume": (cv + pv_) * 100 / out["stock_volume"] if out["stock_volume"] else np.nan})
|
||||
return out
|
||||
|
||||
|
||||
def quality(api: Api, df: pd.DataFrame, session: dt.date) -> tuple[pd.DataFrame, dict]:
|
||||
"""Last OPRA print in 15:45-16:00 vs the indicative quote, near the money."""
|
||||
x = df[df["dte"].between(14, 45) & df["mid"].notna() & df["iv_fwd"].notna()].copy()
|
||||
if x.empty:
|
||||
return x, {}
|
||||
x["m_f"] = (x["strike"] / x["fwd"] - 1).abs()
|
||||
x["m_95"] = (x["strike"] / x["stock_close"] - 0.95).abs()
|
||||
pick = []
|
||||
for _, g in x.groupby("underlying"):
|
||||
e = g.loc[(g["dte"] - 30).abs().idxmin(), "expiry"]
|
||||
g = g[g["expiry"] == e]
|
||||
for typ, col in (("C", "m_f"), ("P", "m_f"), ("P", "m_95")):
|
||||
h = g[g["type"] == typ]
|
||||
if len(h):
|
||||
pick.append(h[col].idxmin())
|
||||
s = x.loc[sorted(set(pick))].copy()
|
||||
t0 = dt.datetime.combine(session, dt.time(15, 30), NY).isoformat()
|
||||
t1 = dt.datetime.combine(session, dt.time(16, 0), NY).isoformat()
|
||||
bars = pd.DataFrame(option_bars(api, s["symbol"].tolist(), t0, t1, "1Min"))
|
||||
if bars.empty:
|
||||
return s.iloc[:0], {}
|
||||
bars["t"] = pd.to_datetime(bars["t"], utc=True).dt.tz_convert(NY)
|
||||
last = bars.sort_values("t").groupby("symbol").tail(1).set_index("symbol")
|
||||
s = s.join(last[["c", "t"]].rename(columns={"c": "print", "t": "print_time"}), on="symbol")
|
||||
cut = pd.Timestamp(dt.datetime.combine(session, dt.time(15, 45), NY))
|
||||
s = s[s["print_time"].notna() & (s["print_time"] >= cut)].copy()
|
||||
if s.empty:
|
||||
return s, {}
|
||||
s["inside"] = (s["print"] >= s["bid"] - 0.005) & (s["print"] <= s["ask"] + 0.005)
|
||||
s["pos"] = (s["print"] - s["mid"]) / (s["ask"] - s["bid"])
|
||||
s["iv_print"] = implied_vol(s["print"], s["fwd"], s["strike"], s["T"], s["type"] == "C", np.exp(-s["r"] * s["T"]))
|
||||
s["iv_err"] = s["iv_print"] - s["iv_fwd"]
|
||||
s["rel_spread"] = (s["ask"] - s["bid"]) / s["mid"]
|
||||
summ = {"q_n": int(len(s)), "q_inside": float(s["inside"].mean()), "q_abs_pos": float(s["pos"].abs().median()),
|
||||
"q_abs_iv_err": float(s["iv_err"].abs().median()), "q_rel_spread": float(s["rel_spread"].median())}
|
||||
cols = ["session", "underlying", "symbol", "type", "expiry", "dte", "strike", "stock_close", "fwd", "bid", "ask",
|
||||
"quote_time", "print", "print_time", "inside", "pos", "iv_fwd", "iv_print", "iv_err", "rel_spread"]
|
||||
return s[cols], summ
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _atomic_csv(df: pd.DataFrame, path: str) -> None:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
df.to_csv(path + ".tmp", index=False, compression="gzip" if path.endswith(".gz") else None)
|
||||
os.replace(path + ".tmp", path)
|
||||
|
||||
|
||||
def _bars_frame(rows: list[dict], meta: pd.DataFrame) -> pd.DataFrame:
|
||||
if not rows:
|
||||
return pd.DataFrame(columns=BAR_COLS)
|
||||
b = pd.DataFrame(rows)
|
||||
b["date"] = b.pop("t").str[:10]
|
||||
b = b.join(meta.set_index("symbol")[["underlying", "type", "expiry", "strike"]], on="symbol")
|
||||
return b[BAR_COLS]
|
||||
|
||||
|
||||
def snapshot(args) -> None:
|
||||
out = args.out or OUT
|
||||
now = dt.datetime.now(NY)
|
||||
api = Api()
|
||||
session = dt.date.fromisoformat(args.session) if args.session else pick_session(api, now, args.force)
|
||||
if session is None:
|
||||
return
|
||||
path = os.path.join(out, "chains", str(session.year), f"chain_{session}.csv.gz")
|
||||
if os.path.exists(path) and not args.force:
|
||||
print(f"{path} exists; nothing to do")
|
||||
return
|
||||
syms = universe()[: args.limit] if args.limit else universe()
|
||||
t0 = time.time()
|
||||
end = min(now - dt.timedelta(minutes=16), dt.datetime.combine(session + dt.timedelta(days=1), dt.time(0), NY))
|
||||
st = stock_bars(api, syms, session.isoformat(), end)
|
||||
st = st[st["date"] == session.isoformat()]
|
||||
_atomic_csv(st.assign(session=session.isoformat()), os.path.join(out, "stock", f"stock_{session}.csv"))
|
||||
px = st.set_index("symbol")[["close", "volume"]].to_dict("index")
|
||||
have = [s for s in syms if s in px]
|
||||
# contracts expiring on the session itself may already be inactive at 16:25,
|
||||
# and 0DTE carries much of the day's volume: list both, as the backfill does
|
||||
meta = pd.DataFrame(list_contracts(api, have, session.isoformat(), (session + dt.timedelta(days=DTE_MAX)).isoformat())
|
||||
+ list_contracts(api, have, session.isoformat(), session.isoformat(), "inactive"))
|
||||
meta = meta.drop_duplicates("symbol")
|
||||
rows, failed = [], []
|
||||
for k, sym in enumerate(syms):
|
||||
if sym not in px:
|
||||
failed.append([sym, "no stock bar"])
|
||||
continue
|
||||
try:
|
||||
rs = chain(api, sym, px[sym]["close"], session)
|
||||
except RuntimeError as exc:
|
||||
failed.append([sym, str(exc)[:120]])
|
||||
continue
|
||||
for r in rs:
|
||||
r.update(stock_close=px[sym]["close"], stock_volume=px[sym]["volume"])
|
||||
rows += rs
|
||||
if (k + 1) % 100 == 0:
|
||||
print(f" {k + 1}/{len(syms)} underlyings, {len(rows)} contracts, {api.n} requests "
|
||||
f"[{time.time() - t0:.0f}s]", flush=True)
|
||||
if not rows:
|
||||
raise SystemExit("no contracts collected")
|
||||
bars_err = None
|
||||
try:
|
||||
bars = _bars_frame(option_bars(api, meta["symbol"].tolist(), session.isoformat(),
|
||||
(session + dt.timedelta(days=1)).isoformat()), meta)
|
||||
bars = bars[bars["date"] == session.isoformat()]
|
||||
except RuntimeError as exc:
|
||||
bars, bars_err = pd.DataFrame(columns=BAR_COLS), str(exc)[:200]
|
||||
_atomic_csv(bars, os.path.join(out, "bars", str(session.year), f"bars_{session}.csv.gz"))
|
||||
df = pd.DataFrame(rows)
|
||||
df = df.join(meta.set_index("symbol")[["oi", "oi_date"]], on="symbol")
|
||||
df = df.join(bars.set_index("symbol")[["c", "v", "vw", "n"]].rename(
|
||||
columns={"c": "opra_close", "v": "opra_volume", "vw": "opra_vwap", "n": "opra_trades"}), on="symbol")
|
||||
df.insert(0, "session", session.isoformat())
|
||||
df["dte"] = (pd.to_datetime(df["expiry"]) - pd.Timestamp(session)).dt.days
|
||||
r, r_date = rate(session)
|
||||
df["r"] = r
|
||||
df = add_iv(df, r)
|
||||
try:
|
||||
qdf, qsum = quality(api, df, session)
|
||||
_atomic_csv(qdf, os.path.join(out, "quality", f"quality_{session}.csv"))
|
||||
except RuntimeError as exc:
|
||||
qsum = {"q_error": str(exc)[:200]}
|
||||
_atomic_csv(df.drop(columns=["mid", "T"]), path)
|
||||
vol = bars.groupby(["underlying", "type"])["v"].sum()
|
||||
summ = pd.DataFrame([summarize(g, vol.get(u, pd.Series(dtype=float)).to_dict())
|
||||
for u, g in df.groupby("underlying")])
|
||||
summ.insert(0, "session", session.isoformat())
|
||||
_atomic_csv(summ, os.path.join(out, "summary", f"summary_{session}.csv"))
|
||||
rec = {"session": session.isoformat(), "started": dt.datetime.fromtimestamp(t0, NY).isoformat(timespec="seconds"),
|
||||
"finished": dt.datetime.now(NY).isoformat(timespec="seconds"), "underlyings": len(syms),
|
||||
"with_chain": int(df["underlying"].nunique()), "contracts": len(df),
|
||||
"valid_iv": int(df["iv_fwd"].notna().sum()), "bars": len(bars), "bars_error": bars_err,
|
||||
"requests": api.n, "r": r, "r_date": r_date, "feed": "indicative", **qsum,
|
||||
"failed": failed[:50], "n_failed": len(failed), "forced": bool(args.force)}
|
||||
with open(os.path.join(out, "manifest.jsonl"), "a") as fh:
|
||||
fh.write(json.dumps(rec) + "\n")
|
||||
print(f"session {session}: {rec['with_chain']}/{len(syms)} underlyings, {len(df)} quoted contracts "
|
||||
f"({rec['valid_iv']} with IV), {len(bars)} OPRA bars, quality {qsum}, {api.n} requests, "
|
||||
f"{time.time() - t0:.0f}s, failed {len(failed)}")
|
||||
|
||||
|
||||
def backfill(args) -> None:
|
||||
out = args.out or OUT
|
||||
api = Api(min_interval=0.4, yield_to_snapshot=True) # ~150 requests a minute
|
||||
end = args.end or (dt.datetime.now(NY).date() - dt.timedelta(days=1)).isoformat()
|
||||
names = args.only or pit_universe(args.start, end)
|
||||
hist = os.path.join(out, "bars_hist")
|
||||
os.makedirs(hist, exist_ok=True)
|
||||
raw = os.path.join(out, "stock_raw.csv.gz")
|
||||
if not os.path.exists(raw) or args.only:
|
||||
st = stock_bars(api, names, args.start, dt.datetime.combine(dt.date.fromisoformat(end), dt.time(23, 59), NY))
|
||||
if not args.only:
|
||||
_atomic_csv(st, raw)
|
||||
print(f"stock bars: {st['symbol'].nunique()} symbols, {len(st)} rows", flush=True)
|
||||
todo = [u for u in names if not os.path.exists(os.path.join(hist, f"{u}.csv.gz"))]
|
||||
print(f"backfill {args.start}..{end}: {len(names)} underlyings, {len(todo)} to do", flush=True)
|
||||
lte_active = (dt.date.fromisoformat(end) + dt.timedelta(days=DTE_MAX)).isoformat()
|
||||
for k, u in enumerate(todo):
|
||||
try:
|
||||
backfill_one(api, u, args.start, end, lte_active, hist, out, k, len(todo))
|
||||
except RuntimeError as exc: # no file is written, so the next run retries it
|
||||
print(f" [{k + 1}/{len(todo)}] {u}: failed, will retry on the next run: {exc}", flush=True)
|
||||
if not args.only and not args.no_panel:
|
||||
volume_panel(args)
|
||||
|
||||
|
||||
def backfill_one(api: Api, u: str, start: str, end: str, lte_active: str, hist: str, out: str, k: int, n: int) -> None:
|
||||
t0, n0 = time.time(), api.n
|
||||
meta = pd.DataFrame(list_contracts(api, [u], start, end, "inactive")
|
||||
+ list_contracts(api, [u], start, lte_active, "active"))
|
||||
frames = []
|
||||
if len(meta):
|
||||
meta = meta.drop_duplicates("symbol")
|
||||
meta["month"] = meta["expiry"].str[:7]
|
||||
for _, g in meta.groupby("month"):
|
||||
first, last = g["expiry"].min(), g["expiry"].max()
|
||||
a = max(start, (dt.date.fromisoformat(first) - dt.timedelta(days=DTE_MAX)).isoformat())
|
||||
b = min(end, last)
|
||||
if a > b:
|
||||
continue
|
||||
f = _bars_frame(option_bars(api, g["symbol"].tolist(), a, (dt.date.fromisoformat(b)
|
||||
+ dt.timedelta(days=1)).isoformat()), g)
|
||||
lo = (pd.to_datetime(f["expiry"]) - pd.Timedelta(days=DTE_MAX)).dt.strftime("%Y-%m-%d")
|
||||
frames.append(f[(f["date"] >= lo) & (f["date"] <= f["expiry"]) & (f["date"] <= end)])
|
||||
bars = pd.concat(frames) if frames else pd.DataFrame(columns=BAR_COLS)
|
||||
_atomic_csv(bars, os.path.join(hist, f"{u}.csv.gz"))
|
||||
rec = {"underlying": u, "contracts": int(len(meta)), "bars": int(len(bars)), "requests": api.n - n0,
|
||||
"seconds": round(time.time() - t0, 1), "start": start, "end": end,
|
||||
"finished": dt.datetime.now(NY).isoformat(timespec="seconds")}
|
||||
with open(os.path.join(out, "backfill_manifest.jsonl"), "a") as fh:
|
||||
fh.write(json.dumps(rec) + "\n")
|
||||
print(f" [{k + 1}/{n}] {u}: {rec['contracts']} contracts, {rec['bars']} bars, "
|
||||
f"{rec['requests']} requests, {rec['seconds']}s (total {api.n})", flush=True)
|
||||
|
||||
|
||||
def volume_panel(args) -> None:
|
||||
"""Daily option volume per underlying from the OPRA bars (backfill + daily
|
||||
files): all strikes with 0-100 days to expiry, and the band the quote
|
||||
snapshot covers (5-100 days, 0.7-1.3 x the unadjusted close)."""
|
||||
out = args.out or OUT
|
||||
stock = [pd.read_csv(os.path.join(out, "stock_raw.csv.gz"))] if os.path.exists(os.path.join(out, "stock_raw.csv.gz")) else []
|
||||
stock += [pd.read_csv(f).drop(columns=["session"], errors="ignore") for f in glob.glob(os.path.join(out, "stock", "*.csv"))]
|
||||
stock = pd.concat(stock).drop_duplicates(["date", "symbol"], keep="last").rename(columns={"symbol": "underlying"})
|
||||
files = sorted(glob.glob(os.path.join(out, "bars_hist", "*.csv.gz")))
|
||||
daily = sorted(glob.glob(os.path.join(out, "bars", "*", "*.csv.gz")))
|
||||
bman = os.path.join(out, "backfill_manifest.jsonl")
|
||||
hist_end = max((json.loads(x)["end"] for x in open(bman) if x.strip()), default="") if os.path.exists(bman) else ""
|
||||
parts = []
|
||||
for f in files + daily:
|
||||
b = pd.read_csv(f)
|
||||
if b.empty:
|
||||
continue
|
||||
if f in daily:
|
||||
b = b[b["date"] > hist_end] # the backfill wins where both exist
|
||||
b = b.merge(stock[["date", "underlying", "close"]], on=["date", "underlying"], how="left")
|
||||
dte = (pd.to_datetime(b["expiry"]) - pd.to_datetime(b["date"])).dt.days
|
||||
band = dte.between(DTE_MIN, DTE_MAX) & (b["strike"] / b["close"]).between(MNY_LO, MNY_HI)
|
||||
b["v_band"] = b["v"].where(band, 0)
|
||||
g = b.groupby(["date", "underlying", "type"])[["v", "v_band"]].sum().unstack("type").fillna(0)
|
||||
g.columns = [f"{a}_{t}" for a, t in g.columns]
|
||||
parts.append(g.reset_index())
|
||||
if not parts:
|
||||
print("no bars yet")
|
||||
return
|
||||
P = pd.concat(parts).groupby(["date", "underlying"]).sum().reset_index()
|
||||
for c in ("v_C", "v_P", "v_band_C", "v_band_P"):
|
||||
if c not in P:
|
||||
P[c] = 0.0
|
||||
P = P.merge(stock[["date", "underlying", "close", "volume"]], on=["date", "underlying"], how="left")
|
||||
P["os_volume"] = (P["v_C"] + P["v_P"]) * 100 / P["volume"]
|
||||
P["os_volume_band"] = (P["v_band_C"] + P["v_band_P"]) * 100 / P["volume"]
|
||||
P["pc_volume"] = P["v_P"] / P["v_C"].replace(0, np.nan)
|
||||
_atomic_csv(P.sort_values(["date", "underlying"]), os.path.join(out, "volume_panel.csv.gz"))
|
||||
print(f"volume panel: {P['underlying'].nunique()} underlyings, {P['date'].nunique()} days "
|
||||
f"({P['date'].min()}..{P['date'].max()}), {len(P)} rows")
|
||||
|
||||
|
||||
def status(args) -> None:
|
||||
out = args.out or OUT
|
||||
man = os.path.join(out, "manifest.jsonl")
|
||||
if os.path.exists(man):
|
||||
recs = [json.loads(x) for x in open(man) if x.strip()]
|
||||
print(f"snapshots: {len(recs)} runs, {len({r['session'] for r in recs})} sessions, "
|
||||
f"{recs[0]['session']}..{recs[-1]['session']}")
|
||||
for r in recs[-5:]:
|
||||
print(f" {r['session']} {r['with_chain']}/{r['underlyings']} underlyings, {r['contracts']} contracts, "
|
||||
f"{r.get('bars', '-')} OPRA bars, failed {r['n_failed']}, {r['started']} -> {r['finished']}")
|
||||
q = [r for r in recs if r.get("q_n")]
|
||||
q = list({r["session"]: r for r in q}.values())
|
||||
if q:
|
||||
ins = float(np.average([r["q_inside"] for r in q], weights=[r["q_n"] for r in q]))
|
||||
err = float(np.median([r["q_abs_iv_err"] for r in q]))
|
||||
print(f"quote quality over {len(q)} sessions: {ins:.1%} of prints inside the indicative quote, "
|
||||
f"median |IV(print) - IV(mid)| {err * 100:.2f} vol points")
|
||||
if len(q) >= QUALITY_MIN_SESSIONS:
|
||||
ok = ins >= QUALITY_INSIDE and err <= QUALITY_IV
|
||||
print(" verdict: indicative quotes adequate" if ok else " verdict: consider the OPRA plan")
|
||||
else:
|
||||
print(f" verdict after {QUALITY_MIN_SESSIONS} sessions")
|
||||
else:
|
||||
print("snapshots: none yet")
|
||||
bman = os.path.join(out, "backfill_manifest.jsonl")
|
||||
if os.path.exists(bman):
|
||||
b = [json.loads(x) for x in open(bman) if x.strip()]
|
||||
print(f"backfill: {len(b)} underlyings done, {sum(x['bars'] for x in b):,} bars, last {b[-1]['underlying']} "
|
||||
f"at {b[-1]['finished']}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
s = sub.add_parser("snapshot")
|
||||
s.add_argument("--limit", type=int, help="first N underlyings only (testing)")
|
||||
s.add_argument("--force", action="store_true", help="record even if late or already recorded")
|
||||
s.add_argument("--session", help="label the snapshot with this session date (testing)")
|
||||
s.add_argument("--out", help="output directory (default data/options/us)")
|
||||
s.set_defaults(fn=snapshot)
|
||||
b = sub.add_parser("backfill")
|
||||
b.add_argument("--start", default=HIST_START)
|
||||
b.add_argument("--end", help="last date (default yesterday)")
|
||||
b.add_argument("--only", nargs="*", help="these underlyings only (testing; no panel)")
|
||||
b.add_argument("--no-panel", action="store_true")
|
||||
b.add_argument("--out")
|
||||
b.set_defaults(fn=backfill)
|
||||
v = sub.add_parser("volume-panel")
|
||||
v.add_argument("--out")
|
||||
v.set_defaults(fn=volume_panel)
|
||||
t = sub.add_parser("status")
|
||||
t.add_argument("--out")
|
||||
t.set_defaults(fn=status)
|
||||
args = ap.parse_args()
|
||||
args.fn(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -13,6 +13,8 @@ through the same engine (PIT members, next-open fills, phase averaging).
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import time
|
||||
@@ -32,13 +34,38 @@ warnings.filterwarnings("ignore")
|
||||
# inputs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ranks_signature(p) -> dict:
|
||||
"""What the cached ranks depend on: the panel's calendar, tickers and
|
||||
membership, and the factor definitions (factors.py as a whole, so a
|
||||
changed helper invalidates every factor)."""
|
||||
with open(os.path.join(os.path.dirname(__file__), "factors.py"), "rb") as fh:
|
||||
src = hashlib.sha256(fh.read()).hexdigest()[:16]
|
||||
cols = hashlib.sha256("\n".join(map(str, p.close.columns)).encode()).hexdigest()[:16]
|
||||
return {"market": p.market, "dates": [str(p.dates[0].date()), str(p.dates[-1].date()), len(p.dates)],
|
||||
"columns": [len(p.close.columns), cols], "member_days": int(p.member.to_numpy().sum()),
|
||||
"factors_py": src}
|
||||
|
||||
|
||||
def factor_ranks(p, market: str, names: list[str] | None = None) -> dict[str, pd.DataFrame]:
|
||||
"""Percentile ranks among PIT members, cached per market (float32)."""
|
||||
"""Percentile ranks among PIT members, cached per market (float32).
|
||||
|
||||
The cache is used only while ranks_{market}.key matches the panel and
|
||||
factors.py; otherwise it is rebuilt (a stale cache silently dropped the
|
||||
Tiingo-backfilled names)."""
|
||||
path = os.path.join("data", "research", f"ranks_{market}.pkl")
|
||||
key = os.path.join("data", "research", f"ranks_{market}.key")
|
||||
sig = ranks_signature(p)
|
||||
cache = {}
|
||||
if os.path.exists(path):
|
||||
with open(path, "rb") as fh:
|
||||
cache = pickle.load(fh)
|
||||
if os.path.exists(path) and os.path.exists(key):
|
||||
with open(key) as fh:
|
||||
fresh = json.load(fh) == sig
|
||||
if fresh:
|
||||
with open(path, "rb") as fh:
|
||||
cache = pickle.load(fh)
|
||||
else:
|
||||
print(f"ranks_{market}.pkl is stale for this panel or factors.py; rebuilding", flush=True)
|
||||
elif os.path.exists(path):
|
||||
print(f"ranks_{market}.pkl has no key file; rebuilding", flush=True)
|
||||
mk = "us" if market == "us" else "cn"
|
||||
names = names or [n for n, f in FACTORS.items() if mk in f.markets]
|
||||
dirty = False
|
||||
@@ -57,6 +84,8 @@ def factor_ranks(p, market: str, names: list[str] | None = None) -> dict[str, pd
|
||||
if dirty:
|
||||
with open(path, "wb") as fh:
|
||||
pickle.dump(cache, fh, protocol=4)
|
||||
with open(key, "w") as fh:
|
||||
json.dump(sig, fh)
|
||||
return {n: cache[n] for n in names if n in cache}
|
||||
|
||||
|
||||
|
||||
200
research/lab/svr_combo.py
Normal file
200
research/lab/svr_combo.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""
|
||||
Does the low short-volume-ratio factor (low_svr21, the only class-A row of
|
||||
research/lab/factor_census.py) improve the momentum strategy or the book?
|
||||
|
||||
PYTHONPATH=. uv run --with numba python -m research.lab.svr_combo
|
||||
|
||||
low_svr21 = -(21-day mean of FINRA REG SHO ShortVolume / TotalVolume),
|
||||
shifted 2 sessions (the day-D file is published after D's close).
|
||||
|
||||
Arms (research engine: PIT S&P 500, next-open fills, 10 bp a side, phase-averaged):
|
||||
base overnight_mom + rec_126 ranks, top 10, every 10 days, buffer 2
|
||||
(the live overnight_recovery_top10)
|
||||
blend_w (1 - w) x base score + w x low_svr21 rank, same construction;
|
||||
w = 1/3 gated, w = 0.25 and 0.5 are the neighbourhood. A member
|
||||
without FINRA data gets a neutral 0.5 SVR rank, so the universe
|
||||
matches base
|
||||
svr_sleeve low_svr21 top 20, every 21 days, buffer 2 (the census construction)
|
||||
Books (overnight_book band_blend: equal shares, 25% band, month-start reset):
|
||||
book3 overnight + lowrisk + trend_v7 (three_sleeve_overnight, the paper test)
|
||||
book4 book3 + svr_sleeve at 1/4 each gated
|
||||
reported book3 with svr_sleeve in place of lowrisk; base + svr_sleeve 50/50
|
||||
|
||||
Window: 2018-10-01 .. 2026-09-24, where SVR covers the members. Halves are
|
||||
split at 2022-07-01. The SVR S&P 500 scan, a veto overlay on the momentum
|
||||
sleeve (research/svr_overlay.py, reverted for a neighbourhood failure) and
|
||||
the census all saw this window before this rule was written. low_svr21 was
|
||||
also picked as the best of 86 census rows. This is a consistency check;
|
||||
only a forward test can be independent.
|
||||
|
||||
The rule is written to prereg.json before the run; the script refuses to
|
||||
overwrite results.json.
|
||||
|
||||
Result (2026-10-12): STOP. Blend: G2 passes; G1 and G3 fail. Sleeve: S1
|
||||
and S2 fail.
|
||||
|
||||
arm / book (2018-10..2026-09) CAGR xSh halves MaxDD alpha vs EW (t)
|
||||
base (top 10 momentum) 27.3% 0.81 0.52 / 1.05 -34.9% +16.4% (1.99)
|
||||
blend w=0.25 28.2% 0.87 0.62 / 1.08 -41.1% +16.2% (2.24)
|
||||
blend w=1/3 27.1% 0.87 0.67 / 1.05 -41.1% +15.0% (2.31)
|
||||
blend w=0.5 23.5% 0.82 0.73 / 0.90 -40.9% +11.8% (2.13)
|
||||
svr_sleeve (top 20) 17.5% 0.80 0.58 / 1.16 -38.7% +5.8% (2.87)
|
||||
book3 (paper-test book) 25.1% 1.49 1.84 / 1.19 -15.0%
|
||||
book4 (+ svr sleeve) 23.2% 1.42 1.49 / 1.34 -21.6%
|
||||
svr in place of lowrisk 28.8% 1.47 1.55 / 1.39 -22.6%
|
||||
base + svr sleeve 50/50 23.0% 0.88 0.56 / 1.24 -35.5%
|
||||
|
||||
- The blend helps everywhere a little: Sharpe +0.06 at w=1/3, vol 33% ->
|
||||
30%, and better in 2018-24. It gives up most of 2026 (+109% -> +66%) and
|
||||
deepens MaxDD by 6pp. That misses the +0.10 bar and the 3pp drawdown bar.
|
||||
- The SVR sleeve correlates 0.50 monthly with the momentum sleeve. Lowrisk
|
||||
and trend already diversify the book, so a fourth sleeve dilutes it
|
||||
(1.49 -> 1.42) and adds 6.6pp of drawdown.
|
||||
- Boundary: 8 years, one regime, and the factor was picked as the best of
|
||||
86 on this same window.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from research.lab.alpha import alpha_stats
|
||||
from research.lab.factor_scan import RESULTS, load
|
||||
from research.lab.factors import FACTORS
|
||||
from research.lab.levered_book import stats, tbill
|
||||
from research.lab.managed_futures_book import band_blend
|
||||
from research.lab.short_volume import factors as svr_factors, load_svr
|
||||
from research.lab.us_sweep import run
|
||||
|
||||
OUT = os.path.join(RESULTS, "svr_combo")
|
||||
WIN = ("2018-10-01", "2026-09-24")
|
||||
HALVES = [("2018-10-01", "2022-06-30"), ("2022-07-01", "2026-09-24")]
|
||||
STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl")
|
||||
OVERNIGHT = os.path.join(RESULTS, "overnight_only", "daily.pkl")
|
||||
W_GATED, W_NEIGHBOURS = 1 / 3, (0.25, 0.5)
|
||||
|
||||
PREREG = {
|
||||
"question": "Does low_svr21 raise the Sharpe of overnight_recovery_top10 (score blend) or of the "
|
||||
"overnight three-sleeve book (extra sleeve) in both halves of 2018-10..2026-09?",
|
||||
"window": WIN, "halves": HALVES,
|
||||
"gates": {
|
||||
"blend (all of)": [
|
||||
"G1 blend w=1/3 excess Sharpe >= base + 0.10 over the window and >= base in each half",
|
||||
"G2 blend w=0.25 and w=0.5 excess Sharpe >= base over the window (no lone spike)",
|
||||
"G3 blend w=1/3 MaxDD >= base MaxDD - 3pp",
|
||||
],
|
||||
"sleeve (all of)": [
|
||||
"S1 book4 excess Sharpe >= book3 + 0.05 in each half",
|
||||
"S2 book4 MaxDD >= book3 MaxDD - 3pp",
|
||||
],
|
||||
},
|
||||
"decision": {
|
||||
"CONTINUE": "an arm passes -> add the daily FINRA short-volume fetch to the live pipeline and run "
|
||||
"the arm forward-only (shadow book) before any live capital",
|
||||
"STOP": "no arm passes -> low_svr21 stays a census row",
|
||||
},
|
||||
"data_status": "window inspected before (SVR scan, veto overlay, census); factor chosen as best of 86",
|
||||
}
|
||||
|
||||
|
||||
def pct(df: pd.DataFrame, member: pd.DataFrame) -> pd.DataFrame:
|
||||
return df.where(member).rank(axis=1, pct=True)
|
||||
|
||||
|
||||
def summary(r: pd.Series, rf: pd.Series, ew: pd.Series) -> dict:
|
||||
r = r.loc[WIN[0]:WIN[1]]
|
||||
s = stats(r, rf)
|
||||
a = alpha_stats(r, ew.reindex(r.index), rf.reindex(r.index))
|
||||
return {"cagr": float(s["cagr"]), "sharpe": float(s["sharpe"]), "vol": float(s["vol"]), "mdd": float(s["mdd"]),
|
||||
"worst_yr": float(s["worst_yr"]), "alpha_ew": float(a["alpha"]), "alpha_ew_t": float(a["t"]),
|
||||
"halves": [float(stats(r.loc[x:y], rf)["sharpe"]) for x, y in HALVES]}
|
||||
|
||||
|
||||
def fmt(k: str, d: dict) -> str:
|
||||
return (f"{k:34s} cagr {d['cagr']*100:5.1f}% xsh {d['sharpe']:5.2f} vol {d['vol']*100:4.1f}% "
|
||||
f"mdd {d['mdd']*100:6.1f}% alpha/EW {d['alpha_ew']*100:+5.1f}% (t {d['alpha_ew_t']:+.2f}) "
|
||||
f"halves {d['halves'][0]:5.2f} {d['halves'][1]:5.2f}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
res_path = os.path.join(OUT, "results.json")
|
||||
if os.path.exists(res_path):
|
||||
raise SystemExit(f"{res_path} exists; this study does not overwrite results")
|
||||
with open(os.path.join(OUT, "prereg.json"), "w") as fh:
|
||||
json.dump(PREREG, fh, indent=2)
|
||||
|
||||
p, _ = load("us")
|
||||
m = p.member
|
||||
house = (pct(FACTORS["overnight_mom"](p), m) + pct(FACTORS["rec_126"](p), m)) / 2
|
||||
svr = -svr_factors(load_svr(p.close))["svr21"].shift(2).reindex(index=p.dates, columns=p.close.columns)
|
||||
svr_r = pct(svr, m).fillna(0.5).where(m) # a member without FINRA data is neutral, not dropped
|
||||
hr = pct(house, m)
|
||||
sp = p.slice("2018-06-01", WIN[1])
|
||||
rf = tbill(p.dates).fillna(0.0)
|
||||
ret = p.close.pct_change(fill_method=None)
|
||||
ew = ret.where(m.shift(1, fill_value=False)).mean(axis=1)
|
||||
live = dict(top_n=10, every=10, buffer=2.0)
|
||||
subs = [WIN]
|
||||
|
||||
S = {"base": run(sp, house.where(m), subs, **live)["_rets"]}
|
||||
for w in (W_NEIGHBOURS[0], W_GATED, W_NEIGHBOURS[1]):
|
||||
S[f"blend_w{w:.2f}"] = run(sp, ((1 - w) * hr + w * svr_r).where(m), subs, **live)["_rets"]
|
||||
S["svr_sleeve"] = run(sp, svr.where(m), subs, top_n=20, every=21, buffer=2.0)["_rets"]
|
||||
S = pd.DataFrame(S).loc[WIN[0]:WIN[1]]
|
||||
|
||||
R = pd.read_pickle(STREAMS)
|
||||
O = pd.read_pickle(OVERNIGHT)
|
||||
B = pd.DataFrame({"overnight": O["sleeve_score"], "lowrisk": R["lowrisk"], "trend_v7": R["trend_v7"]})
|
||||
B = B.join(S[["svr_sleeve", "base"]], how="inner").loc[WIN[0]:WIN[1]]
|
||||
books = {
|
||||
"book3": band_blend(B[["overnight", "lowrisk", "trend_v7"]], rf)[0],
|
||||
"book4": band_blend(B[["overnight", "lowrisk", "trend_v7", "svr_sleeve"]], rf)[0],
|
||||
"book3_svr_for_lowrisk": band_blend(B[["overnight", "svr_sleeve", "trend_v7"]], rf)[0],
|
||||
"base+svr_sleeve 50/50": band_blend(B[["base", "svr_sleeve"]], rf)[0],
|
||||
}
|
||||
res = {"arms": {}, "books": {}}
|
||||
print("=== arms")
|
||||
for k in S:
|
||||
res["arms"][k] = summary(S[k], rf, ew)
|
||||
print(fmt(k, res["arms"][k]))
|
||||
print("=== books")
|
||||
for k, r in books.items():
|
||||
res["books"][k] = summary(r, rf, ew)
|
||||
print(fmt(k, res["books"][k]))
|
||||
mon = (1 + S).resample("ME").prod() - 1
|
||||
res["monthly_corr_svr_sleeve_vs_base"] = float(mon["svr_sleeve"].corr(mon["base"]))
|
||||
print(f"monthly corr svr_sleeve vs base {res['monthly_corr_svr_sleeve_vs_base']:.2f}")
|
||||
yr = (1 + S).groupby(S.index.year).prod() - 1
|
||||
res["yearly"] = {k: {int(y): float(v) for y, v in yr[k].items()} for k in yr}
|
||||
print("yearly %:\n", (yr * 100).round(1).to_string())
|
||||
|
||||
a, b = res["arms"], res["books"]
|
||||
g = a[f"blend_w{W_GATED:.2f}"]
|
||||
checks = {
|
||||
"G1": g["sharpe"] >= a["base"]["sharpe"] + 0.10 and all(g["halves"][i] >= a["base"]["halves"][i] for i in range(2)),
|
||||
"G2": all(a[f"blend_w{w:.2f}"]["sharpe"] >= a["base"]["sharpe"] for w in W_NEIGHBOURS),
|
||||
"G3": g["mdd"] >= a["base"]["mdd"] - 0.03,
|
||||
"S1": all(b["book4"]["halves"][i] >= b["book3"]["halves"][i] + 0.05 for i in range(2)),
|
||||
"S2": b["book4"]["mdd"] >= b["book3"]["mdd"] - 0.03,
|
||||
}
|
||||
passing = [arm for arm, keys in (("blend", ("G1", "G2", "G3")), ("sleeve", ("S1", "S2")))
|
||||
if all(checks[k] for k in keys)]
|
||||
res["verdict"] = {"checks": checks, "passing": passing, "decision": "CONTINUE" if passing else "STOP"}
|
||||
print(f"checks {checks} -> {res['verdict']['decision']} {passing}")
|
||||
with open(res_path, "w") as fh:
|
||||
json.dump(res, fh, indent=2)
|
||||
S.join(pd.DataFrame(books)).to_pickle(os.path.join(OUT, "returns.pkl"))
|
||||
from research.lab.trials import append_trials
|
||||
append_trials(os.path.join(RESULTS, "trials_us.jsonl"),
|
||||
[{"name": f"svr_combo:{k}", **{x: v for x, v in d.items() if x != "halves"}}
|
||||
for grp in ("arms", "books") for k, d in res[grp].items()],
|
||||
{"study": "svr_combo", "role": "preregistered consistency check"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user