From 92af4e5ac043c25e361c2deed36b753edeb50eac Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 8 Oct 2026 09:48:05 -0700 Subject: [PATCH] feat(research): Tiingo backfill of delisted names; honest PIT panels Yahoo purges delisted members: the S&P research panel was missing 209 ever-members (AGN/CELG/ESRX/ATVI/TWTR/XLNX/...), the production store 89, and the S&P 400/600 panels ~180 more. The gap inflated momentum-family factors (overnight_mom holdout 16.9->14.8%/yr, rec_126 14.7->10.7 on the research panel; factor_panel_check). research/lab/fetch_tiingo.py pulls full EOD histories (free tier: 500 unique symbols/month, 50/hour; paced, resume-safe, key never logged). research/lab/tiingo_merge.py overlays the caches on the research S&P panel at build time (>=50% PIT-coverage gate, median-overlap cross-check, stub-column replacement, rename splices GPS<-GAP, DISCA<-WBD, WLTW<-WTW, ADS<-BFH, CCE<-CCEP, ESV<-VAL, FBHS<-FBIN, HFC<-DINO) and can apply the same to the production store (--production [--apply]; data/us*.csv, .bak-tiingo backups, under the panel lock). load_us() rebuilds when the overlay changes (_tiingo_sig); fetch_us_mid.load_bars falls back to converted CSVs. Round 1 (2026-10-08): 165/209 S&P names and 183 mid/small PIT members restored (2011-19 snapshots were 15-20% short of PIT members). prod_impact A/B on registered replays: overnight_recovery_top10 23.8%/0.83/-38.3% vs 27.8%/0.88/-47.3% backfill-off; lowrisk_top20 6.1%/0.58 vs 8.6%/0.60 - honest long returns with shallower drawdowns; trend_rider_v10 (ETF path) unchanged at 55.0%/1.64. Official precompute refreshed (backup at data/precomputed/backtest_results.json.pretiingo). midsmall_scan on the corrected panel: no factor beats the EW benchmark (0.60 holdout Sharpe vs 0.51-0.52 best) - no mid/small sleeve. --- .gitignore | 1 + CLAUDE.md | 27 +++ research/factor_panel_check.py | 73 ++++++++ research/lab/fetch_tiingo.py | 122 ++++++++++++ research/lab/fetch_us_mid.py | 9 +- research/lab/panel.py | 27 ++- research/lab/panel_gaps.py | 134 +++++++++++++ research/lab/tiingo_merge.py | 331 +++++++++++++++++++++++++++++++++ research/midsmall_rebuild.py | 50 +++++ research/midsmall_scan.py | 131 +++++++++++++ research/prod_impact.py | 75 ++++++++ 11 files changed, 977 insertions(+), 3 deletions(-) create mode 100644 research/factor_panel_check.py create mode 100644 research/lab/fetch_tiingo.py create mode 100644 research/lab/panel_gaps.py create mode 100644 research/lab/tiingo_merge.py create mode 100644 research/midsmall_rebuild.py create mode 100644 research/midsmall_scan.py create mode 100644 research/prod_impact.py diff --git a/.gitignore b/.gitignore index fc7c307..01d143d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ Thumbs.db data/.*.lock data/backup_*/ data/*_unavailable.json +data/*.bak-tiingo # Local secrets, logs and agent scratch output .env diff --git a/CLAUDE.md b/CLAUDE.md index a8965cb..286c1c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,6 +81,30 @@ PYTHONPATH=. uv run python -m research.lab.sector_cycle # busin uv run python -m research.lab.fetch_wiki_attention && PYTHONPATH=. uv run --with numba python -m research.lab.attention # Wikipedia-pageview attention PYTHONPATH=. uv run --with numba python -m research.lab.concentration # industry caps on the momentum sleeve PYTHONPATH=. uv run python -m research.lab.universe_midcap # overnight_recovery_top10 on S&P 500 ∪ S&P 400 (preregistered; FAIL 2026-10-08) +uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs) +PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] # separate S&P 400 sleeve vs IJH (preregistered; all REJECT 2026-10-08) +PYTHONPATH=. uv run python -m research.lab.fetch_shortvolume # FINRA REG SHO daily short volume (~8y archive, resume-safe) +PYTHONPATH=. uv run python -m research.lab.short_volume [us|midsmall|all] # SVR (short-volume ratio) IC / quintile scan on PIT panels +PYTHONPATH=. uv run python -m research.lab.fetch_fails # SEC fails-to-deliver archives (2018+, resume-safe) +PYTHONPATH=. uv run python -m research.lab.fails [us|midsmall|all] # FTD intensity scans (value/shares per ADV$, PIT +21d publication lag) +PYTHONPATH=. uv run python -m research.lab.fetch_13d [--quarters N] # EDGAR quarterly index: SC 13D/13G rows (activist/passive stakes) +PYTHONPATH=. uv run python -m research.lab.fetch_13d_targets [--limit N] # subject company of each SC 13D (index lists the filer) +PYTHONPATH=. uv run python -m research.lab.activists [us|midsmall|all] # 13D event study: market-adjusted CARs, momentum/SVR splits +PYTHONPATH=. uv run python -m research.lab.activist_strategy [midsmall|us] # 13D event sleeve (lab engine; dead in 2021-26 — research only) +PYTHONPATH=. uv run python -m research.lab.fetch_13f # SEC DERA 13F structured data sets (2016q1-2023q4), per-CUSIP aggregates +PYTHONPATH=. uv run python -m research.lab.inst_flow [us|midsmall] # institutional q/q flow factor scan (IC t 2.0 but Q5-Q1 spread nil) +PYTHONPATH=. uv run python -m research.lab.fetch_gdelt && PYTHONPATH=. uv run python -m research.lab.news_volume # GDELT news volume (API rate-limited to ~1 query/10-15s; sample only) +PYTHONPATH=. uv run python -m research.svr_overlay # SVR-veto overlay on overnight_recovery_top10 (lab) + +# Tiingo backfill (delisted names Yahoo purged; keys from TIINGO_API_KEY, never logged) +PYTHONPATH=. uv run python -m research.lab.fetch_tiingo --file list.txt --min-interval 75 # EOD backfill (free tier: 500 symbols/month, 50/hour; cache = resume) +PYTHONPATH=. uv run python -m research.lab.panel_gaps # price gaps vs PIT membership (both panels); writes data/research/tiingo_plan.json +PYTHONPATH=. uv run python -m research.lab.tiingo_merge # report: overlay caches onto the research S&P panel (auto at panel build) +PYTHONPATH=. uv run python -m research.lab.tiingo_merge --production [--apply] # merge into data/us*.csv (backups .bak-tiingo; --apply writes) +PYTHONPATH=. uv run python -m research.lab.tiingo_merge --yahoo-mid # convert caches to yahoo_mid format (fetch_us_mid.load_bars falls back to it) +PYTHONPATH=. uv run python -m research.factor_panel_check # pre-vs-post overlay S&P factor comparison +PYTHONPATH=. uv run python -m research.midsmall_scan --panel both # mid/small factor scan: Yahoo-only vs Tiingo-extended panel +PYTHONPATH=. uv run python -m research.prod_impact [STRATEGY ...] # A/B registered replays: backfilled store vs .bak-tiingo snapshot # Setup uv sync # Install/sync dependencies @@ -126,7 +150,10 @@ uv run python -m unittest discover tests - `recovery_momentum.py` — Recovery (price/63d low) + 12-1mo momentum composite. Best US performer. - `robust_momentum.py` — Bar strategies from the 2026-09 PIT research (report: https://claude.ai/artifact/Mo9nrzs5SWj5FfyshERyWS): US `overnight_recovery` (overnight-return momentum + distance from 126d low), CN `lowrisk_momentum` (amplitude-cut momentum + low beta + price-volume corr + momentum/vol) and close-only `lowbeta_sharpe_mom`. Top-N with rank hysteresis, rebalance every 10 days, weights drift between rebalances. `low_risk` (US `lowrisk_top20`: idiosyncratic vol + low beta, top 20, monthly) is the defensive sleeve of the 2026-09-29 US research (report: https://claude.ai/artifact/VGhZvKkS9UBYZkLeqGjcNC). - `sleeve_portfolio.py` — Several strategies in one account with fixed capital shares. `us_three_sleeve` = 1/3 each of `overnight_recovery_top10`, `lowrisk_top20`, `trend_rider_v7_vt24` (a mixed stock+ETF strategy); sleeve shares drift with their returns and reset at a month start once any is >25% off. 2011-2026 research: Sharpe 1.21 vs 0.88-1.03 for the sleeves alone. Research finding behind it: choosing factors by their recent record (rotation, factor momentum, last year's winner) did not beat a fixed composite; construction knobs (N, weighting, dynamic N) barely matter. On synthetic 3x ETFs from 1990 (`research/lab/letf_history.py`), TrendRider V7's 2016-2026 record does not hold: 1990-99 Sharpe 0.2-0.3, 2003-07 negative, 1990-2026 about 0.7 with MaxDD -58% (vt24) to -69% (vt36). +- `trend_rider_v9.py` — V7 pipeline (V3 regime + vol target + profit-take) plus an instrument trailing stop: while risk-on, a close 20% below the held leg's 30-day high (PIT, through t-1) switches to the risk-off basket immediately; re-entry still needs the normal regime confirmation. Registered as `trend_rider_v9` (VT34, the default) and `trend_rider_v9_vt24/_vt32/_vt36`. Engine window 2017-12→2026-10, $10k, $2/order: `trend_rider_v9` 52.3%/yr Sharpe 1.55 MaxDD -29.3%, `_vt36` 52.5%/1.53, `_vt32` 51.5%/1.57, `_vt24` 42.4%/1.58. 2010-2026 live-book simulation ($2/order): every tier improves on V7 (e.g. vt34: 34.4%/1.14 vs vt32 32.2%/1.10); 2010-2016 blocks are essentially unchanged. +- `trend_rider_v10` (same file) — V9 plus a refined profit-take: clear at +28% (not 30%), restore band 8pp (re-enter sooner on dips), 10-day re-clear lockout after a restore (stops the 2022-style clear/restore churn on GLD/DBC). Registered as `trend_rider_v10` (VT34), `_vt32`, `_vt36`. Engine window to 2026-10-07: v10 55.9%/1.66/MaxDD -27.4%, v10_vt32 55.0%/1.68, v10_vt36 56.2%/1.63 — beats every other registered strategy on both CAGR and Sharpe (previous best: v9_vt36 52.5%, v9_vt24 1.58). 2010-2026 live-book simulation confirms robustness: the PT refinement lifts every 3-4y block's Sharpe (2010-13 0.83→0.92, 2016-21 1.74→1.89, 2021-26 1.38→1.43; full period 36.8%/1.22 vs 34.4%/1.14 for plain V9-trail). Research trail: `research/tr9_{lab,run,long,trials,audit}.py`; rejected knobs are logged in `data/research/results/trials_us.jsonl` (macro throttles on VIX term/credit, RSP breadth, instrument MA exits, fast re-entry, SOXL/TECL/TMF/SQQQ baskets). - `qqq_trend.py` — `QQQVolTrend`: QQQ exposure = 1{QQQ > 200-day average} × min(1.5, 30% / EWMA vol), held as TQQQ (E/3) + SHY. 1990-2026 on synthetic ETFs: 16.0% vs QQQ 14.9%, MaxDD -37% vs -83%; 2011-2026 on real ETFs it trails QQQ (17.5% vs 19.1%) with a smaller drawdown. `us_momentum_qqq_trend` (in `sleeve_portfolio.py`) holds it half and half with `overnight_recovery_top10`. Research behind it (`research/lab/qqq_scan.py`, `beat_qqq.py`): measured as alpha against QQQ with a 2006-16 design / 2017-26 holdout split, momentum top 10 is the only stock selection whose alpha held (+10%/yr, t 1.4); single factors, walk-forward ML, a mega-cap universe, SEC insider trades, 10-K/10-Q text similarity, earnings-announcement returns, industry momentum and a PIT S&P 400/600 universe (`panel_mid.py`) showed none. In the web engine (2016-09 to 2026-09, $2/order) `us_momentum_qqq_trend` earns 21.0% vs QQQ 20.9%, MaxDD -32%. +- `svr_momentum.py` — RESEARCH ONLY (not registered): `overnight_recovery_top10` + a FINRA short-volume veto (drop top-70%-quantile SVR names). The 2026-10-07 window looked good (Sharpe 0.87→0.89, MaxDD -47%→-39%) but the quantile neighborhood killed it — 0.60/0.75/0.80 vetoes all score below baseline. A lone spike; registered and reverted the same day. The raw cross-sectional reading (SVR negative predictor, `research/lab/short_volume.py`) stands; the concentrated-portfolio exclusion rule does not. - `agent_review.py` — `overnight_recovery_agent_top10`: overnight_recovery's top 50, re-ranked each day by an LLM agent's scores (−2..+2 from filings, fundamentals, demand-chain logic, event risk; `final = quant percentile + 0.25 × score`), vetoes for five event types, top 10 on the base strategy's rebalance rows and hysteresis. Score files in `data/agent/us/` are stamped by `agent_review.py submit` and steer a session only if stamped before its 16:00 New York close; without one the strategy holds exactly what `overnight_recovery_top10` holds, so the two books' difference measures the agent. `AGENT_REVIEW_STRATEGIES` in trader.py keeps it US-only and out of `precompute_backtest.py` (it has no history before its first score file). Research behind the design (2026-09-30): picking 10 of the quant top 50 at random earns 13.4%/yr vs 26.2% for the top 10 (2006-2026), and an LLM cannot be backtested before its training cutoff, so the record starts 2026-09-29. - `factor_combo.py` — Configurable factor-combination strategies with daily/weekly/biweekly/monthly rebalancing. US champions: `rec_mfilt+deep_upvol`, `ma200+mom7m+rec126`, `rec_mfilt+ma200`, `mom7m+rec126`. CN champions: `up_cap+quality_mom`, `down_resil+qual_mom`, `rec63+mom_gap`, `up_cap+mom_gap` — these were mined on today's CSI 300 members; on PIT members they earn ~3-7% a year (2016-2026). All registered in trader.py as `fc_{signal}_{freq}` (e.g., `fc_rec_mfilt_deep_upvol_monthly`). 32 new strategies total. diff --git a/research/factor_panel_check.py b/research/factor_panel_check.py new file mode 100644 index 0000000..cb28013 --- /dev/null +++ b/research/factor_panel_check.py @@ -0,0 +1,73 @@ +"""Pre-vs-post Tiingo overlay factor check for the S&P panel. + +Builds the panel twice in-process — Yahoo-only (pre, survivorship-biased) +and with the Tiingo overlay (post) — and runs the sleeve-relevant factors +through the standard top-N phase-averaged backtest on both. The delta is the +survivorship effect on the factors behind the live stock sleeves +(overnight_recovery: overnight_mom + rec_126; lowrisk: ivol + low_beta). + + PYTHONPATH=. uv run python -m research.factor_panel_check +""" +from __future__ import annotations + +import os +import time +import warnings + +import pandas as pd + +import research.lab.panel as pnl +from research.lab.factor_scan import PERIODS, phase_avg +from research.lab.factors import FACTORS +from research.lab.report import forward_returns, ic_series, ic_summary + +warnings.filterwarnings("ignore") + +NAMES = ["overnight_mom", "rec_126", "mom_12_1", "low_vol", + "ivol", "low_beta", "sharpe_mom", "amp_cut_mom"] +RESULTS = os.path.join("data", "research", "results") + + +def scan(p, label: str) -> pd.DataFrame: + fwd5 = forward_returns(p, 5) + rows = [] + for name in NAMES: + t0 = time.time() + s = FACTORS[name](p) + row = {"panel": label, "factor": name} + for per, (a, b) in PERIODS["us"].items(): + st = phase_avg(p, s, a, b, 20, 5) + ic = ic_summary(ic_series(s.loc[a:b], fwd5.loc[a:b], p.member.loc[a:b], 5)) + row.update({f"{per}_cagr": st["cagr"], f"{per}_sharpe": st["sharpe"], + f"{per}_ic5": ic["ic"], f"{per}_ic5_t": ic["t"]}) + rows.append(row) + print(f"[{label}] {name:14s} D {row['design_cagr']:6.1%}/{row['design_sharpe']:.2f} " + f"H {row['holdout_cagr']:6.1%}/{row['holdout_sharpe']:.2f} [{time.time() - t0:.0f}s]", + flush=True) + return pd.DataFrame(rows) + + +def main() -> None: + post = pnl.load_us() + orig = pnl.apply_sp500 + pnl.apply_sp500 = lambda raw, **kw: raw + try: + pre = pnl._build_us() + finally: + pnl.apply_sp500 = orig + a = scan(pre, "pre") + b = scan(post, "post") + df = pd.concat([a, b], ignore_index=True) + os.makedirs(RESULTS, exist_ok=True) + df.to_csv(os.path.join(RESULTS, "factor_panel_check.csv"), index=False) + m = a.merge(b, on="factor", suffixes=("_pre", "_post")) + print("\nfactor D_cagr pre->post D_sh pre->post H_cagr pre->post H_sh pre->post") + for _, r in m.iterrows(): + print(f"{r['factor']:14s} {r['design_cagr_pre']:7.1%}->{r['design_cagr_post']:6.1%} " + f"{r['design_sharpe_pre']:5.2f}->{r['design_sharpe_post']:5.2f} " + f"{r['holdout_cagr_pre']:7.1%}->{r['holdout_cagr_post']:6.1%} " + f"{r['holdout_sharpe_pre']:5.2f}->{r['holdout_sharpe_post']:5.2f}") + + +if __name__ == "__main__": + main() diff --git a/research/lab/fetch_tiingo.py b/research/lab/fetch_tiingo.py new file mode 100644 index 0000000..ea7da95 --- /dev/null +++ b/research/lab/fetch_tiingo.py @@ -0,0 +1,122 @@ +"""Fetch EOD prices from Tiingo (free tier: 500 unique symbols/month, 50/hour). + +Used for: (1) completing delisted/missing names in the research panels, +(2) independent cross-validation of the Yahoo-sourced closes. The key comes +from TIINGO_API_KEY (environment or .env) and is never logged or written. + +Writes data/research/tiingo/.csv (resume-safe; a cached symbol never +counts against the monthly quota again). Columns: as-traded open/high/low/ +close/volume (Tiingo's close is NOT split-adjusted), split/dividend-adjusted +adjOpen..adjVolume, plus splitFactor and divCash. + + PYTHONPATH=. uv run python -m research.lab.fetch_tiingo TICKER [TICKER ...] + PYTHONPATH=. uv run python -m research.lab.fetch_tiingo --file symbols.txt + PYTHONPATH=. uv run python -m research.lab.fetch_tiingo --file l.txt --min-interval 75 +""" +from __future__ import annotations + +import argparse +import os +import re +import sys +import time + +import pandas as pd +import requests + +DIR = os.path.join("data", "research", "tiingo") +BASE = "https://api.tiingo.com/tiingo/daily/{t}/prices" +KEEP = ("date", "open", "high", "low", "close", "volume", + "adjOpen", "adjHigh", "adjLow", "adjClose", "adjVolume", + "splitFactor", "divCash") + + +def api_key() -> str: + key = os.environ.get("TIINGO_API_KEY") + if not key and os.path.exists(".env"): + for line in open(".env"): + m = re.match(r"\s*TIINGO_API_KEY\s*=\s*(.+)\s*$", line) + if m: + key = m.group(1).strip().strip("'\"") + break + if not key: + raise SystemExit("TIINGO_API_KEY not found (env or .env)") + return key + + +def fetch(ticker: str, key: str, start: str = "1990-01-01") -> pd.DataFrame | None: + """One symbol's full EOD history; None when Tiingo does not have it (404). + Network errors back off and retry; a hard failure skips the ticker.""" + for attempt in range(4): + try: + r = requests.get(BASE.format(t=ticker.lower()), + params={"startDate": start, "format": "json"}, + headers={"Content-Type": "application/json", + "Authorization": f"Token {key}"}, timeout=60) + except requests.RequestException as exc: + print(f"{ticker}: {type(exc).__name__}; retry {attempt + 1}", flush=True) + time.sleep(20 * (attempt + 1)) + continue + if r.status_code == 429: + print(f"{ticker}: rate limited; waiting 15 min", flush=True) + time.sleep(900) + continue + if r.status_code in (402, 403): # monthly symbol quota exhausted + raise SystemExit(f"{ticker}: HTTP {r.status_code} — quota reached; " + "remaining tickers stay uncached for the next run") + if r.status_code == 404: + return None + r.raise_for_status() + d = r.json() + if not d: + return None + df = pd.DataFrame(d) + df["date"] = pd.to_datetime(df["date"]).dt.tz_localize(None) + keep = [c for c in KEEP if c in df.columns] + return df[keep].set_index("date") + raise RuntimeError(f"{ticker}: network retries exhausted") + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("tickers", nargs="*") + ap.add_argument("--file", help="one ticker per line") + ap.add_argument("--min-interval", type=float, default=75.0, + help="seconds between network fetches (free tier allows 50/hour)") + ap.add_argument("--start", default="1990-01-01") + args = ap.parse_args() + tickers = [l.strip() for l in open(args.file) if l.strip()] if args.file else args.tickers + if not tickers: + raise SystemExit("usage: fetch_tiingo TICKER ... | --file list.txt") + + os.makedirs(DIR, exist_ok=True) + key = api_key() + todo = [t for t in tickers if not os.path.exists(os.path.join(DIR, f"{t.upper()}.csv"))] + print(f"{len(tickers)} tickers, {len(todo)} to fetch", flush=True) + fetched = failed = 0 + last = 0.0 + for i, t in enumerate(todo): + wait = args.min_interval - (time.time() - last) + if wait > 0: + time.sleep(wait) + last = time.time() + try: + df = fetch(t, key, args.start) + except RuntimeError: + print(f"[{i + 1}/{len(todo)}] {t:8s} network fail; retried next run", flush=True) + continue + if df is None: + failed += 1 + print(f"[{i + 1}/{len(todo)}] {t:8s} not found", flush=True) + continue + df.to_csv(os.path.join(DIR, f"{t.upper()}.csv")) + fetched += 1 + eta_min = (len(todo) - i - 1) * args.min_interval / 60 + print(f"[{i + 1}/{len(todo)}] {t:8s} {len(df)} rows " + f"{df.index.min().date()}..{df.index.max().date()} (ETA {eta_min:.0f}m)", + flush=True) + print(f"done: {fetched} fetched, {failed} not found", flush=True) + + +if __name__ == "__main__": + main() diff --git a/research/lab/fetch_us_mid.py b/research/lab/fetch_us_mid.py index ac46a2d..88b44b1 100644 --- a/research/lab/fetch_us_mid.py +++ b/research/lab/fetch_us_mid.py @@ -56,6 +56,7 @@ import pandas as pd ROOT = os.path.join("data", "research", "ext") HOLD_DIR = os.path.join(ROOT, "ishares") PX_DIR = os.path.join(ROOT, "yahoo_mid") +TIINGO_DIR = os.path.join(ROOT, "tiingo_mid") HOLDINGS = os.path.join(HOLD_DIR, "holdings.csv") FUNDS = {"IJH": 239763, "IJR": 239774} # S&P MidCap 400, S&P SmallCap 600 ETFS = ["IJH", "IJR", "MDY", "SLY", "IWM", "SPY", "QQQ", "BIL", "SHY"] @@ -394,10 +395,14 @@ def fetch_prices(start: str = "2005-01-01", retry_missing: bool = False) -> None def load_bars(t: str) -> pd.DataFrame | None: - """Adjusted OHLCV (as yfinance's auto_adjust=True) plus the unadjusted close.""" + """Adjusted OHLCV (as yfinance's auto_adjust=True) plus the unadjusted + close. Falls back to a Tiingo-converted CSV (see lab.tiingo_merge).""" path = _px_path(t) if not os.path.exists(path): - return None + alt = os.path.join(TIINGO_DIR, os.path.basename(path)) + if not os.path.exists(alt): + return None + path = alt h = pd.read_csv(path, index_col=0, parse_dates=True) h = h[~h.index.duplicated(keep="last")].sort_index() ratio = (h["Adj Close"] / h["Close"]).where(h["Close"] > 0) diff --git a/research/lab/panel.py b/research/lab/panel.py index 5754378..ead25ac 100644 --- a/research/lab/panel.py +++ b/research/lab/panel.py @@ -18,6 +18,7 @@ import numpy as np import pandas as pd from research.lab.engine import Costs, Panel +from research.lab.tiingo_merge import apply_sp500 RAW_DIR = os.path.join("data", "research") US_ETFS = {"SPY", "RSP", "QQQ", "SHY", "BIL", "IEF", "TLT", "GLD", @@ -54,6 +55,7 @@ def _us_membership(dates: pd.DatetimeIndex, tickers: list[str]) -> pd.DataFrame: def _build_us() -> Panel: with open(os.path.join(RAW_DIR, "us_ohlcv.pkl"), "rb") as fh: raw = pickle.load(fh) + raw = apply_sp500(raw) close = raw["close"] # keep rows where most listed tickers printed (drops stray holiday rows) alive = close.notna().sum(axis=1) @@ -92,8 +94,31 @@ def _us_sectors(stocks: list[str]) -> pd.Series: return sec.reindex(stocks) +def _tiingo_sig() -> list: + d = os.path.join(RAW_DIR, "tiingo") + if not os.path.isdir(d): + return [0, 0] + files = [f for f in os.listdir(d) if f.endswith(".csv")] + mtime = max((os.path.getmtime(os.path.join(d, f)) for f in files), default=0) + return [len(files), round(mtime, 3)] + + def load_us() -> Panel: - return _cached("us", _build_us) + """Cached US panel, rebuilt whenever the Tiingo overlay changes.""" + path = os.path.join(RAW_DIR, "us.panel.pkl") + key = os.path.join(RAW_DIR, "us.panel.key") + sig = _tiingo_sig() + if os.path.exists(path) and os.path.exists(key): + with open(key) as fh: + if json.load(fh) == sig: + with open(path, "rb") as fh: + return pickle.load(fh) + p = _build_us() + with open(path, "wb") as fh: + pickle.dump(p, fh) + with open(key, "w") as fh: + json.dump(sig, fh) + return p # --------------------------------------------------------------------------- diff --git a/research/lab/panel_gaps.py b/research/lab/panel_gaps.py new file mode 100644 index 0000000..a9d7a8f --- /dev/null +++ b/research/lab/panel_gaps.py @@ -0,0 +1,134 @@ +"""Quantify price gaps against PIT membership in the US research panels and +emit a Tiingo backfill plan (free tier: 500 unique symbols/month, 50/hour). + +For S&P 500 ever-members: names with no data, a late start or an early end +relative to their membership intervals, in both the production books +(data/us.csv) and the research panel (data/research/us_ohlcv.pkl). For the +mid/small panels: the Yahoo-missing tickers (data/research/ext/yahoo_mid/ +missing.json) ranked by snapshot weight and recency, since only the top of +that list fits the monthly quota. + +Writes data/research/tiingo_plan.json. + + PYTHONPATH=. uv run python -m research.lab.panel_gaps +""" +from __future__ import annotations + +import json +import os +import pickle + +import numpy as np +import pandas as pd + +TOL_DAYS = 45 +PLAN = os.path.join("data", "research", "tiingo_plan.json") + + +def sp500_intervals() -> dict: + with open(os.path.join("data", "sp500_history.json")) as fh: + return json.load(fh)["intervals"] + + +def classify(px: pd.DataFrame, intervals: dict) -> pd.DataFrame: + """Per ever-member: membership vs data presence within the panel window.""" + rows = [] + pstart, pend = px.index[0], px.index[-1] + for t, iv in intervals.items(): + iv = [(pd.Timestamp(s) if s else None, pd.Timestamp(e) if e else None) for s, e in iv] + mstart = max(min((s for s, _ in iv if s), default=pstart), pstart) + mend = min(max((e for _, e in iv if e), default=pend), pend) + if mend <= pstart or mstart >= pend: + continue + col = px[t].dropna() if t in px.columns else pd.Series(dtype=float) + first = col.index[0] if len(col) else None + last = col.index[-1] if len(col) else None + n_mem = int(((px.index >= mstart) & (px.index < mend)).sum()) + n_px = int(((col.index >= mstart) & (col.index < mend)).sum()) if len(col) else 0 + flags = [] + if not len(col): + flags.append("no_data") + else: + if first > mstart + pd.Timedelta(days=TOL_DAYS): + flags.append("late_start") + if last < mend - pd.Timedelta(days=TOL_DAYS): + flags.append("early_end") + rows.append(dict(ticker=t, m_first=mstart.date(), m_last=mend.date(), + px_first=first.date() if first is not None else None, + px_last=last.date() if last is not None else None, + n_mem=n_mem, n_px=n_px, + cov=round(n_px / n_mem, 3) if n_mem else np.nan, + flags=",".join(flags))) + return pd.DataFrame(rows) + + +def mid_small_plan() -> pd.DataFrame: + """Rank the Yahoo-missing mid/small tickers: total snapshot weight, with + recent years weighted fully and older ones decayed.""" + from research.lab.fetch_us_mid import load_holdings + + miss_path = os.path.join("data", "research", "ext", "yahoo_mid", "missing.json") + missing = set(json.load(open(miss_path))) + h = load_holdings() + h = h[h["ticker"].isin(missing)] + if h.empty: + return h + recency = pd.Series(1.0, index=h.index) + recency[h["date"] < "2014-01-01"] = 0.25 + recency[(h["date"] >= "2014-01-01") & (h["date"] < "2019-01-01")] = 0.5 + h = h.assign(score=h["weight"] * recency) + g = h.groupby("ticker") + out = pd.DataFrame({ + "n_snaps": g.size(), + "w_sum": g["weight"].sum().round(4), + "score": g["score"].sum().round(4), + "first": g["date"].min().dt.date, + "last": g["date"].max().dt.date, + "name": g["name"].first(), + }).sort_values("score", ascending=False) + return out + + +def main() -> None: + intervals = sp500_intervals() + + import data_manager + prod = data_manager.load_raw("us") + print(f"production data/us.csv: {prod.shape[1]} cols {prod.index[0].date()}..{prod.index[-1].date()}") + + from research.lab.tiingo_merge import apply_sp500 + with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh: + raw = pickle.load(fh) + res = apply_sp500(raw, verbose=False)["close"] + print(f"research us_ohlcv.pkl + tiingo: {res.shape[1]} cols " + f"{res.index[0].date()}..{res.index[-1].date()}") + + plan: dict[str, list[str]] = {} + for label, px in (("production", prod), ("research", res)): + df = classify(px, intervals) + bad = df[df["flags"] != ""].sort_values(["n_mem"], ascending=False) + print(f"\n[{label}] ever-members with intervals in window: {len(df)}; with flags: {len(bad)}") + for fl in ("no_data", "late_start", "early_end"): + sub = bad[bad["flags"].str.contains(fl)] + print(f" {fl}: {len(sub)}") + cols = ["ticker", "m_first", "m_last", "px_first", "px_last", "cov"] + if len(sub): + print(sub[cols].head(25).to_string(index=False)) + names = sorted(set(bad[bad["flags"].str.contains("no_data|early_end")]["ticker"])) + plan[f"sp500_{label}"] = names + print(f" -> {len(names)} no_data/early_end names") + + ms = mid_small_plan() + print(f"\n[mid/small] missing tickers with snapshot weight: {len(ms)}") + print(ms.head(30).to_string()) + print(f" weight coverage reachable in quota (top 500): " + f"{ms['w_sum'].head(500).sum() / ms['w_sum'].sum():.1%}") + plan["mid_small"] = list(ms.index) + + with open(PLAN, "w") as fh: + json.dump(plan, fh, indent=1) + print(f"\nSaved {PLAN}") + + +if __name__ == "__main__": + main() diff --git a/research/lab/tiingo_merge.py b/research/lab/tiingo_merge.py new file mode 100644 index 0000000..a6639d8 --- /dev/null +++ b/research/lab/tiingo_merge.py @@ -0,0 +1,331 @@ +"""Overlay Tiingo EOD caches (data/research/tiingo/) onto the US research panels. + +S&P 500 -- ``apply_sp500(raw)``: fills the Yahoo-built panel's gaps, whole +missing columns for delisted names and late tails, keeping Yahoo cells where +both sources have data. Safety per ticker: >=50% of its PIT membership +sessions must be covered; where both sources overlap (>=20 sessions) the +median Tiingo/Yahoo close ratio must sit within 2% (same total-return +adjustment basis; a recycled ticker fails this). Two rename splices +(GPS<-GAP, DISCA<-WBD) backfill pre-rename eras from the successor ticker. +Read-only on the Yahoo artifact: deleting data/research/tiingo/ reverts it. +The merge prints an audit report incl. terminal gaps (last bar vs membership +end), which is the delisted-terminal check. + +Mid/small -- ``write_yahoo_mid()``: converts the caches to the yahoo_mid CSV +format (split-adjusted OHLC with a Stock Splits column so load_bars can +recover the as-traded close, Adj Close as served) under +data/research/ext/tiingo_mid/, which fetch_us_mid.load_bars falls back to. + + PYTHONPATH=. uv run python -m research.lab.tiingo_merge --check + PYTHONPATH=. uv run python -m research.lab.tiingo_merge --yahoo-mid +""" +from __future__ import annotations + +import json +import os +import sys + +import numpy as np +import pandas as pd + +DIR = os.path.join("data", "research", "tiingo") +MID_DIR = os.path.join("data", "research", "ext", "tiingo_mid") +FIELDS = ("open", "high", "low", "close", "volume") +SPLICE = {"GPS": "GAP", "DISCA": "WBD"} +MIN_COVERAGE = 0.5 +MIN_OVERLAP = 20 +RATIO_TOL = 0.02 +TERMINAL_GAP_DAYS = 45 +# Pure ticker changes where the successor's series is the same security's +# continuous price history (Yahoo/Tiingo migrate it on rename). NOT for +# merger-shell cases (RTN->RTX pre-2020 is United Technologies, not Raytheon). +SPLICE.update({"WLTW": "WTW", "ADS": "BFH", "CCE": "CCEP", + "ESV": "VAL", "FBHS": "FBIN", "HFC": "DINO"}) + +_intervals_cache: dict | None = None + + +def _intervals() -> dict: + global _intervals_cache + if _intervals_cache is None: + with open(os.path.join("data", "sp500_history.json")) as fh: + _intervals_cache = json.load(fh)["intervals"] + return _intervals_cache + + +def _cached(t: str) -> pd.DataFrame: + h = pd.read_csv(os.path.join(DIR, f"{t.upper()}.csv"), index_col=0, parse_dates=True) + return h[~h.index.duplicated(keep="last")].sort_index() + + +def cached_tickers() -> list[str]: + if not os.path.isdir(DIR): + return [] + return sorted(f[:-4] for f in os.listdir(DIR) if f.endswith(".csv")) + + +def _frame(t: str) -> pd.DataFrame: + """Adjusted OHLCV in the panel convention (auto_adjust=True).""" + h = _cached(t) + out = pd.DataFrame({f: h["adj" + f[0].upper() + f[1:]] for f in FIELDS}) + return out + + +def _member_mask(t: str, dates: pd.DatetimeIndex) -> np.ndarray: + m = np.zeros(len(dates), bool) + for s, e in _intervals().get(t, []): + si = dates.searchsorted(pd.Timestamp(s) if s else dates[0], side="left") + ei = dates.searchsorted(pd.Timestamp(e) if e else dates[-1] + pd.Timedelta(days=1), + side="left") + m[si:ei] = True + return m + + +def _coverage(t: str, frame: pd.DataFrame, dates: pd.DatetimeIndex) -> float: + mask = _member_mask(t, dates) + if not mask.any(): + return 0.0 + have = frame["close"].reindex(dates).notna().to_numpy() + return float((mask & have).sum()) / float(mask.sum()) + + +def _cross_check(yahoo: pd.Series, tiingo: pd.Series) -> float | None: + """Median Tiingo/Yahoo close ratio over the overlap; None if too small.""" + y = yahoo.reindex(tiingo.index) + both = y.notna() & tiingo["close"].notna() + if both.sum() < MIN_OVERLAP: + return None + return float((tiingo.loc[both, "close"] / y[both]).median()) + + +def apply_sp500(raw: dict, *, verbose: bool = True) -> dict: + """Return a copy of the raw panel dict with Tiingo cells filled in.""" + dates = raw["close"].index + out = {k: v.copy() for k, v in raw.items()} + merged, extended, replaced, skipped, rejected, spliced, terminals = [], [], [], [], [], [], [] + frames = {t: _frame(t) for t in cached_tickers()} + + def fill_column(t: str, frame: pd.DataFrame, mask: np.ndarray | None = None): + added = t not in out["close"].columns + for f in FIELDS: + col = frame[f].reindex(dates) + if mask is not None: + col = col.where(mask) + if added: + out[f][t] = col + else: + out[f][t] = out[f][t].where(out[f][t].notna(), col) + return added + + for t, frame in frames.items(): + cov = _coverage(t, frame, dates) + if cov < MIN_COVERAGE: + skipped.append((t, round(cov, 3))) + continue + replace = False + if t in out["close"].columns: + mask = _member_mask(t, dates) + have = out["close"][t].reindex(dates).notna().to_numpy() + if mask.any() and (mask & have).sum() / mask.sum() < MIN_COVERAGE: + # a recycled ticker or a broken stub sits in the column: the + # full series wins wherever it has data + replace = True + else: + med = _cross_check(out["close"][t], frame) + if med is not None and not (1 - RATIO_TOL < med < 1 + RATIO_TOL): + rejected.append((t, round(med, 4))) + continue + if replace: + for f in FIELDS: + col = frame[f].reindex(dates) + out[f][t] = col.where(col.notna(), out[f][t]) + replaced.append(t) + else: + added = fill_column(t, frame) + (merged if added else extended).append(t) + # terminal check: last bar vs membership end within the panel window + col = out["close"][t].dropna() + ends = [pd.Timestamp(e) if e else dates[-1] + pd.Timedelta(days=1) + for _, e in _intervals().get(t, [])] + mend = min(max(ends), dates[-1]) if ends else None + if mend is not None and len(col): + gap = (mend - col.index[-1]).days + if gap > TERMINAL_GAP_DAYS: + terminals.append((t, str(col.index[-1].date()), str(mend.date()), gap)) + + for old, new in SPLICE.items(): + frame = frames.get(new) + if frame is None and new in out["close"].columns: + frame = pd.DataFrame({f: out[f][new] for f in FIELDS}) + if frame is None: + continue + mask = _member_mask(old, dates) + if not mask.any(): + continue + have = out["close"][old].reindex(dates).notna().to_numpy() if old in out["close"] else np.zeros(len(dates), bool) + if (mask & have).sum() / mask.sum() >= MIN_COVERAGE: + continue + fill_column(old, frame, mask) + spliced.append(f"{old}<-{new}") + + if verbose: + print(f"tiingo overlay: {len(frames)} cached files; merged {len(merged)}, " + f"extended {len(extended)}, replaced {len(replaced)}, spliced {len(spliced)}, " + f"skipped {len(skipped)}, rejected {len(rejected)}") + if merged: + print(f" merged: {', '.join(merged)}") + if extended: + print(f" extended: {', '.join(extended)}") + if replaced: + print(f" replaced (column was a stub/recycled): {', '.join(replaced)}") + if spliced: + print(f" spliced: {', '.join(spliced)}") + if skipped: + print(f" skipped (coverage<{MIN_COVERAGE}): " + + ", ".join(f"{t}:{c}" for t, c in skipped[:30]) + + (" ..." if len(skipped) > 30 else "")) + if rejected: + print(f" rejected (ratio off >{RATIO_TOL:.0%}): " + + ", ".join(f"{t}:{r}" for t, r in rejected)) + if terminals: + print(f" terminal gaps >{TERMINAL_GAP_DAYS}d: " + + ", ".join(f"{t} last={d} member_end={m} ({g}d)" for t, d, m, g in terminals)) + return out + + +PROD_FIELDS = ("open", "high", "low", "volume") + + +def _production_plan() -> tuple: + """Which cached tickers to merge into the production store, and how.""" + import data_manager + base = data_manager.load_raw("us") + if base is None: + raise FileNotFoundError("data/us.csv not found") + dates = base.index + frames = {t: _frame(t) for t in cached_tickers()} + add, extend, replace, skip, rejected = [], [], [], [], [] + for t, fr in frames.items(): + cov = _coverage(t, fr, dates) + if cov < MIN_COVERAGE: + skip.append((t, round(cov, 3))) + continue + if t in base.columns: + mask = _member_mask(t, dates) + have = base[t].reindex(dates).notna().to_numpy() + if mask.any() and (mask & have).sum() / mask.sum() < MIN_COVERAGE: + replace.append(t) + continue + med = _cross_check(base[t], fr) + if med is not None and not (1 - RATIO_TOL < med < 1 + RATIO_TOL): + rejected.append((t, round(med, 4))) + continue + n = int(base[t].notna().sum()) + (extend if n else add).append(t) + else: + add.append(t) + for old, new in SPLICE.items(): + if old in add: + continue + source = frames.get(new) + if source is None and new in base.columns: + source = pd.DataFrame({f: data_manager.load_raw("us", f)[new] + for f in ("close",) + PROD_FIELDS}) + if source is None: + continue + mask = _member_mask(old, dates) + if not mask.any(): + continue + have = (base[old].reindex(dates).notna().to_numpy() if old in base.columns + else np.zeros(len(dates), bool)) + if (mask & have).sum() / mask.sum() < MIN_COVERAGE: + add.append(old) + fr = source.reindex(dates) + frames[old] = fr.where(np.broadcast_to(mask[:, None], fr.shape)) + return base, frames, add, extend, replace, skip, rejected + + +def apply_production(*, dry_run: bool = False) -> None: + """Merge Tiingo frames into data/us*.csv (backup .bak-tiingo on first write).""" + import data_manager + from data_manager import _data_path, _panel_lock, _write_atomic + base, frames, add, extend, replace, skip, rejected = _production_plan() + names = add + extend + replace + print(f"production merge: {len(frames)} cached; add {len(add)}, extend {len(extend)}, " + f"replace {len(replace)}, skip {len(skip)}, rejected {len(rejected)}") + if add: + print(f" add: {', '.join(add)}") + if extend: + print(f" extend: {', '.join(extend)}") + if replace: + print(f" replace: {', '.join(replace)}") + if rejected: + print(f" rejected: {', '.join(f'{t}:{r}' for t, r in rejected)}") + if dry_run or not names: + return + with _panel_lock("us"): + fields = {f: data_manager.load_raw("us", f) for f in PROD_FIELDS} + repl = set(replace) + for f in FIELDS: + panel = base if f == "close" else fields[f] + if panel is None: + continue + for t in names: + col = frames[t][f].reindex(panel.index) + if t not in panel.columns: + panel[t] = col + elif t in repl: + panel[t] = col.where(col.notna(), panel[t]) + else: + panel[t] = panel[t].where(panel[t].notna(), col) + path = _data_path("us", f) + if not os.path.exists(path + ".bak-tiingo"): + import shutil + shutil.copy2(path, path + ".bak-tiingo") + _write_atomic(panel, path) + print(f"production store updated for {len(names)} tickers " + f"(backups: data/us*.csv.bak-tiingo)") + + +def write_yahoo_mid(tickers: list[str] | None = None) -> None: + """Convert caches to yahoo_mid-format CSVs (split-adjusted OHLC + Stock + Splits + as-traded close recoverable via load_bars).""" + os.makedirs(MID_DIR, exist_ok=True) + names = tickers if tickers is not None else cached_tickers() + written = 0 + for t in names: + h = _cached(t) + if not {"close", "adjClose", "splitFactor"}.issubset(h.columns): + print(f"{t}: missing columns; refetch with the current fetch_tiingo") + continue + splits = h["splitFactor"].replace(0, 1.0).fillna(1.0) + later = splits[::-1].cumprod()[::-1].shift(-1).fillna(1.0) + out = pd.DataFrame({ + "Open": h["open"] / later, "High": h["high"] / later, + "Low": h["low"] / later, "Close": h["close"] / later, + "Adj Close": h["adjClose"], "Volume": h["adjVolume"], + "Dividends": h.get("divCash", pd.Series(0.0, index=h.index)).fillna(0.0), + "Stock Splits": splits, + }) + out.index.name = "Date" + out.to_csv(os.path.join(MID_DIR, f"{t.upper()}.csv")) + written += 1 + print(f"wrote {written} converted CSVs to {MID_DIR}") + + +def main() -> None: + if "--yahoo-mid" in sys.argv: + args = [a for a in sys.argv[1:] if not a.startswith("--")] + write_yahoo_mid(args or None) + return + if "--production" in sys.argv: + apply_production(dry_run="--apply" not in sys.argv) + return + import pickle + with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh: + raw = pickle.load(fh) + apply_sp500(raw) + + +if __name__ == "__main__": + main() diff --git a/research/midsmall_rebuild.py b/research/midsmall_rebuild.py new file mode 100644 index 0000000..58ac3be --- /dev/null +++ b/research/midsmall_rebuild.py @@ -0,0 +1,50 @@ +"""Convert mid/small Tiingo caches, rebuild the PIT panels, report the +survivorship change (pre-Tiingo backup vs rebuilt member counts per year). + + PYTHONPATH=. uv run python -m research.midsmall_rebuild +""" +from __future__ import annotations + +import os +import pickle +import shutil + +import pandas as pd + + +def main() -> None: + stage = [l.strip() for l in open("data/research/tiingo_midsmall.txt")][:230] + cached = [n for n in stage if os.path.exists(f"data/research/tiingo/{n}.csv")] + print(f"mid/small tickers: {len(stage)} listed, {len(cached)} cached", flush=True) + + from research.lab.tiingo_merge import write_yahoo_mid + write_yahoo_mid(cached) + + for f in ("us_mid_bars.pkl", "us_mid.panel.pkl", "us_small.panel.pkl", "us_midsmall.panel.pkl"): + p = os.path.join("data", "research", f) + if os.path.exists(p): + os.remove(p) + print(f"removed {p}", flush=True) + + from research.lab.panel_mid import load_us_mid + new = load_us_mid("midsmall") + with open("data/research/us_midsmall.panel.pre_tiingo.pkl", "rb") as fh: + old = pickle.load(fh) + + n, o = new.member.sum(axis=1), old.member.sum(axis=1) + comp = pd.DataFrame({"post": n.resample("YE").last(), "pre": o.resample("YE").last()}).dropna() + comp["delta"] = comp["post"] - comp["pre"] + comp.index = comp.index.year + print("\nPIT members per year (pre -> post):") + print(comp.to_string()) + from research.lab.fetch_us_mid import load_bars + restored = [t for t in cached if t in new.close.columns + and new.member[t].sum() > 0 and load_bars(t) is not None + and t not in old.close.columns] + print(f"\nnew PIT-covered names not in old panel: {len(restored)}") + print(new.close.shape, flush=True) + print("REBUILD DONE", flush=True) + + +if __name__ == "__main__": + main() diff --git a/research/midsmall_scan.py b/research/midsmall_scan.py new file mode 100644 index 0000000..b179377 --- /dev/null +++ b/research/midsmall_scan.py @@ -0,0 +1,131 @@ +"""Mid/small factor scan with a survivorship experiment. + +Compact top-N next-open backtests (full phase averaging, 20bp/side) on the +PIT S&P 400/600 union Panel, run against BOTH builds: the Yahoo-only panel +(pre-Tiingo backup, delisted names silently missing) and the Tiingo-extended +rebuild. A factor whose conclusion flips between the two was a survivorship +artifact; one that survives both is actionable. + +Windows: design 2007-01..2016-12, holdout 2017-01..2026-09. Scores are +computed on the full history and the index is masked to PIT members, so +pre-admission bars inform indicators but never selection. + +The overnight composite approximates the production robust_momentum sleeve +(overnight-return momentum + distance from the 126d low), not its exact code. + + PYTHONPATH=. uv run python -m research.midsmall_scan --panel both +""" +from __future__ import annotations + +import argparse +import os +import pickle +import warnings + +import numpy as np +import pandas as pd + +from research.lab.engine import simulate +from research.lab.report import forward_returns, ic_series, ic_summary, stats + +warnings.filterwarnings("ignore") + +DESIGN = ("2007-01-01", "2016-12-31") +HOLDOUT = ("2017-01-01", "2026-09-24") +RESULTS = os.path.join("data", "research", "results") + + +def load(panel: str): + if panel == "new": + from research.lab.panel_mid import load_us_mid + return load_us_mid("midsmall") + with open(os.path.join("data", "research", "us_midsmall.panel.pre_tiingo.pkl"), "rb") as fh: + return pickle.load(fh) + + +def factors(p) -> dict: + c = p.close.where(p.member) + o = p.open.where(p.member) + vol = c.pct_change(fill_method=None).rolling(60, min_periods=40).std() + onr = ((o / c.shift(1) - 1).where(lambda x: x.abs() < 0.5) + .rolling(63, min_periods=40).mean()) + dist126 = c / c.rolling(126, min_periods=100).min() - 1 + hi252 = c / c.rolling(252, min_periods=200).max() + pr = lambda x: x.rank(axis=1, pct=True) + return { + "mom_12_1": c.shift(21) / c.shift(252) - 1, + "mom_6_1": c.shift(21) / c.shift(126) - 1, + "rev_1m": -(c / c.shift(21) - 1), + "rev_1w": -(c / c.shift(5) - 1), + "lowvol": -vol, + "hi252": hi252, + "dist126": dist126, + "onr63": onr, + "orm_comp": pr(onr) + pr(dist126), + } + + +def phase_avg(p, s, window, top_n, every) -> dict: + a, b = window + w0 = p.dates.searchsorted(pd.Timestamp(a)) + sp = p.slice(p.dates[max(w0 - 5, 0)], b) + rows = [] + for off in range(every): + r = simulate(sp, s, top_n=top_n, every=every, offset=off, warmup=5) + rows.append(stats(r.equity.loc[a:b], r.turnover)) + return pd.DataFrame(rows).mean().to_dict() + + +def run(panel: str, top_n: int, every: int) -> pd.DataFrame: + p = load(panel) + fwd5 = forward_returns(p, 5) + rows = [] + for name, s in factors(p).items(): + row = {"panel": panel, "factor": name, "top_n": top_n, "every": every} + for wname, window in (("design", DESIGN), ("holdout", HOLDOUT)): + st = phase_avg(p, s, window, top_n, every) + ic = ic_summary(ic_series(s.loc[window[0]:window[1]], fwd5.loc[window[0]:window[1]], + p.member.loc[window[0]:window[1]], 5)) + row.update({f"{wname}_cagr": st["cagr"], f"{wname}_sharpe": st["sharpe"], + f"{wname}_mdd": st["mdd"], f"{wname}_turn": st["turnover"], + f"{wname}_ic5": ic["ic"], f"{wname}_ic_t": ic["t"]}) + rows.append(row) + print(f"{name:10s} D cagr {row['design_cagr']:6.1%} sh {row['design_sharpe']:5.2f} " + f"ic {row['design_ic5']:+.3f} | H cagr {row['holdout_cagr']:6.1%} " + f"sh {row['holdout_sharpe']:5.2f} ic {row['holdout_ic5']:+.3f}", flush=True) + return pd.DataFrame(rows) + + +def ew_universe(p) -> None: + r = p.close.pct_change(fill_method=None).where(p.member.shift(1, fill_value=False)) + ew = (1 + r.mean(axis=1).fillna(0.0)).cumprod() + b = p.benchmark + for wname, (a, bnd) in (("design", DESIGN), ("holdout", HOLDOUT)): + se = stats(ew.loc[a:bnd]) + sb = stats(b.loc[a:bnd]) + print(f" EW members {wname}: cagr {se['cagr']:6.1%} sh {se['sharpe']:5.2f} mdd {se['mdd']:6.1%}" + f" | ETF blend: cagr {sb['cagr']:6.1%} sh {sb['sharpe']:5.2f}") + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--panel", default="both", choices=["old", "new", "both"]) + ap.add_argument("--top-n", type=int, default=20) + ap.add_argument("--every", type=int, default=10) + args = ap.parse_args() + panels = ["old", "new"] if args.panel == "both" else [args.panel] + out = [] + for panel in panels: + print(f"\n=== {panel} panel ({args.top_n} names, every {args.every}) ===") + p = load(panel) + ew_universe(p) + out.append(run(panel, args.top_n, args.every)) + df = pd.concat(out, ignore_index=True) + os.makedirs(RESULTS, exist_ok=True) + path = os.path.join(RESULTS, f"midsmall_scan_{args.every}.csv") + df.to_csv(path, index=False) + print(f"saved {path}") + + +if __name__ == "__main__": + main() diff --git a/research/prod_impact.py b/research/prod_impact.py new file mode 100644 index 0000000..0909031 --- /dev/null +++ b/research/prod_impact.py @@ -0,0 +1,75 @@ +"""A/B the Tiingo backfill's effect on registered replays. + +Runs the requested strategies through backtest_runner on the current +production store, then again with the pre-backfill snapshot (data/us*.csv +.bak-tiingo) swapped in, and prints CAGR / Sharpe / MaxDD deltas. Files are +swapped under the data_manager panel lock and restored afterwards. + + PYTHONPATH=. uv run python -m research.prod_impact [STRATEGY ...] +""" +from __future__ import annotations + +import os +import shutil +import sys + +import metrics +import backtest_runner as br +import data_manager + +FILES = ["us.csv", "us_open.csv", "us_high.csv", "us_low.csv", "us_volume.csv"] +START = "2018-01-01" + + +def run(names: list[str]) -> dict: + r = br.run_market("us", names, None, 10_000.0, 0.001, 2.0, refresh=False, start=START) + out = {} + for s in r["strategy_runs"]: + eq = s["equity"] + out[s["name"]] = { + "cagr": metrics.annualized_return(eq), + "sharpe": metrics.sharpe_ratio(eq.pct_change().dropna()), + "mdd": metrics.max_drawdown(eq), + } + return out + + +def swap(pre_backfill: bool) -> None: + """Swap the .bak-tiingo snapshot in or out (live files preserved as .merged-tmp).""" + for f in FILES: + live = os.path.join("data", f) + bak = live + ".bak-tiingo" + tmp = live + ".merged-tmp" + if not os.path.exists(bak) and not os.path.exists(tmp): + continue + if pre_backfill: + if os.path.exists(bak): + shutil.copy2(live, tmp) + shutil.copy2(bak, live) + else: + if os.path.exists(tmp): + shutil.copy2(tmp, live) + os.remove(tmp) + + +def main() -> None: + names = sys.argv[1:] or ["overnight_recovery_top10", "lowrisk_top20"] + with data_manager._panel_lock("us"): + print("=== merged (Tiingo backfill) ===") + a = run(names) + swap(pre_backfill=True) + try: + print("=== pre-backfill snapshot ===") + b = run(names) + finally: + with data_manager._panel_lock("us"): + swap(pre_backfill=False) + print("\nstrategy cagr% sharpe mdd%") + for n in a: + print(f"{n:34s} {a[n]['cagr']*100:6.1f}->{b[n]['cagr']*100:6.1f} " + f"{a[n]['sharpe']:5.2f}->{b[n]['sharpe']:5.2f} " + f"{a[n]['mdd']*100:6.1f}->{b[n]['mdd']*100:6.1f}") + + +if __name__ == "__main__": + main()