feat(research): Tiingo backfill of delisted names; honest PIT panels

Yahoo purges delisted members: the S&P research panel was missing 209 ever-members (AGN/CELG/ESRX/ATVI/TWTR/XLNX/...), the production store 89, and the S&P 400/600 panels ~180 more. The gap inflated momentum-family factors (overnight_mom holdout 16.9->14.8%/yr, rec_126 14.7->10.7 on the research panel; factor_panel_check). research/lab/fetch_tiingo.py pulls full EOD histories (free tier: 500 unique symbols/month, 50/hour; paced, resume-safe, key never logged). research/lab/tiingo_merge.py overlays the caches on the research S&P panel at build time (>=50% PIT-coverage gate, median-overlap cross-check, stub-column replacement, rename splices GPS<-GAP, DISCA<-WBD, WLTW<-WTW, ADS<-BFH, CCE<-CCEP, ESV<-VAL, FBHS<-FBIN, HFC<-DINO) and can apply the same to the production store (--production [--apply]; data/us*.csv, .bak-tiingo backups, under the panel lock). load_us() rebuilds when the overlay changes (_tiingo_sig); fetch_us_mid.load_bars falls back to converted CSVs. Round 1 (2026-10-08): 165/209 S&P names and 183 mid/small PIT members restored (2011-19 snapshots were 15-20% short of PIT members). prod_impact A/B on registered replays: overnight_recovery_top10 23.8%/0.83/-38.3% vs 27.8%/0.88/-47.3% backfill-off; lowrisk_top20 6.1%/0.58 vs 8.6%/0.60 - honest long returns with shallower drawdowns; trend_rider_v10 (ETF path) unchanged at 55.0%/1.64. Official precompute refreshed (backup at data/precomputed/backtest_results.json.pretiingo). midsmall_scan on the corrected panel: no factor beats the EW benchmark (0.60 holdout Sharpe vs 0.51-0.52 best) - no mid/small sleeve.
2026-10-08 09:48:05 -07:00
parent 2f754998a3
commit 92af4e5ac0
11 changed files with 977 additions and 3 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -55,6 +55,7 @@ Thumbs.db
 data/.*.lock
 data/backup_*/
 data/*_unavailable.json
+data/*.bak-tiingo

 # Local secrets, logs and agent scratch output
 .env
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -81,6 +81,30 @@ PYTHONPATH=. uv run python -m research.lab.sector_cycle                  # busin
 uv run python -m research.lab.fetch_wiki_attention && PYTHONPATH=. uv run --with numba python -m research.lab.attention   # Wikipedia-pageview attention
 PYTHONPATH=. uv run --with numba python -m research.lab.concentration     # industry caps on the momentum sleeve
 PYTHONPATH=. uv run python -m research.lab.universe_midcap              # overnight_recovery_top10 on S&P 500 ∪ S&P 400 (preregistered; FAIL 2026-10-08)
+uv run python -m research.lab.fetch_us_fundamentals --universe mid      # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs)
+PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4]  # separate S&P 400 sleeve vs IJH (preregistered; all REJECT 2026-10-08)
+PYTHONPATH=. uv run python -m research.lab.fetch_shortvolume              # FINRA REG SHO daily short volume (~8y archive, resume-safe)
+PYTHONPATH=. uv run python -m research.lab.short_volume [us|midsmall|all] # SVR (short-volume ratio) IC / quintile scan on PIT panels
+PYTHONPATH=. uv run python -m research.lab.fetch_fails                   # SEC fails-to-deliver archives (2018+, resume-safe)
+PYTHONPATH=. uv run python -m research.lab.fails [us|midsmall|all]       # FTD intensity scans (value/shares per ADV$, PIT +21d publication lag)
+PYTHONPATH=. uv run python -m research.lab.fetch_13d [--quarters N]            # EDGAR quarterly index: SC 13D/13G rows (activist/passive stakes)
+PYTHONPATH=. uv run python -m research.lab.fetch_13d_targets [--limit N]      # subject company of each SC 13D (index lists the filer)
+PYTHONPATH=. uv run python -m research.lab.activists [us|midsmall|all]        # 13D event study: market-adjusted CARs, momentum/SVR splits
+PYTHONPATH=. uv run python -m research.lab.activist_strategy [midsmall|us]   # 13D event sleeve (lab engine; dead in 2021-26 — research only)
+PYTHONPATH=. uv run python -m research.lab.fetch_13f                        # SEC DERA 13F structured data sets (2016q1-2023q4), per-CUSIP aggregates
+PYTHONPATH=. uv run python -m research.lab.inst_flow [us|midsmall]          # institutional q/q flow factor scan (IC t 2.0 but Q5-Q1 spread nil)
+PYTHONPATH=. uv run python -m research.lab.fetch_gdelt && PYTHONPATH=. uv run python -m research.lab.news_volume  # GDELT news volume (API rate-limited to ~1 query/10-15s; sample only)
+PYTHONPATH=. uv run python -m research.svr_overlay                        # SVR-veto overlay on overnight_recovery_top10 (lab)
+
+# Tiingo backfill (delisted names Yahoo purged; keys from TIINGO_API_KEY, never logged)
+PYTHONPATH=. uv run python -m research.lab.fetch_tiingo --file list.txt --min-interval 75  # EOD backfill (free tier: 500 symbols/month, 50/hour; cache = resume)
+PYTHONPATH=. uv run python -m research.lab.panel_gaps                    # price gaps vs PIT membership (both panels); writes data/research/tiingo_plan.json
+PYTHONPATH=. uv run python -m research.lab.tiingo_merge                  # report: overlay caches onto the research S&P panel (auto at panel build)
+PYTHONPATH=. uv run python -m research.lab.tiingo_merge --production [--apply]  # merge into data/us*.csv (backups .bak-tiingo; --apply writes)
+PYTHONPATH=. uv run python -m research.lab.tiingo_merge --yahoo-mid      # convert caches to yahoo_mid format (fetch_us_mid.load_bars falls back to it)
+PYTHONPATH=. uv run python -m research.factor_panel_check                # pre-vs-post overlay S&P factor comparison
+PYTHONPATH=. uv run python -m research.midsmall_scan --panel both        # mid/small factor scan: Yahoo-only vs Tiingo-extended panel
+PYTHONPATH=. uv run python -m research.prod_impact [STRATEGY ...]        # A/B registered replays: backfilled store vs .bak-tiingo snapshot

 # Setup
 uv sync                                  # Install/sync dependencies
@@ -126,7 +150,10 @@ uv run python -m unittest discover tests
 - `recovery_momentum.py` — Recovery (price/63d low) + 12-1mo momentum composite. Best US performer.
 - `robust_momentum.py` — Bar strategies from the 2026-09 PIT research (report: https://claude.ai/artifact/Mo9nrzs5SWj5FfyshERyWS): US `overnight_recovery` (overnight-return momentum + distance from 126d low), CN `lowrisk_momentum` (amplitude-cut momentum + low beta + price-volume corr + momentum/vol) and close-only `lowbeta_sharpe_mom`. Top-N with rank hysteresis, rebalance every 10 days, weights drift between rebalances. `low_risk` (US `lowrisk_top20`: idiosyncratic vol + low beta, top 20, monthly) is the defensive sleeve of the 2026-09-29 US research (report: https://claude.ai/artifact/VGhZvKkS9UBYZkLeqGjcNC).
 - `sleeve_portfolio.py` — Several strategies in one account with fixed capital shares. `us_three_sleeve` = 1/3 each of `overnight_recovery_top10`, `lowrisk_top20`, `trend_rider_v7_vt24` (a mixed stock+ETF strategy); sleeve shares drift with their returns and reset at a month start once any is >25% off. 2011-2026 research: Sharpe 1.21 vs 0.88-1.03 for the sleeves alone. Research finding behind it: choosing factors by their recent record (rotation, factor momentum, last year's winner) did not beat a fixed composite; construction knobs (N, weighting, dynamic N) barely matter. On synthetic 3x ETFs from 1990 (`research/lab/letf_history.py`), TrendRider V7's 2016-2026 record does not hold: 1990-99 Sharpe 0.2-0.3, 2003-07 negative, 1990-2026 about 0.7 with MaxDD -58% (vt24) to -69% (vt36).
+- `trend_rider_v9.py` — V7 pipeline (V3 regime + vol target + profit-take) plus an instrument trailing stop: while risk-on, a close 20% below the held leg's 30-day high (PIT, through t-1) switches to the risk-off basket immediately; re-entry still needs the normal regime confirmation. Registered as `trend_rider_v9` (VT34, the default) and `trend_rider_v9_vt24/_vt32/_vt36`. Engine window 2017-12→2026-10, $10k, $2/order: `trend_rider_v9` 52.3%/yr Sharpe 1.55 MaxDD -29.3%, `_vt36` 52.5%/1.53, `_vt32` 51.5%/1.57, `_vt24` 42.4%/1.58. 2010-2026 live-book simulation ($2/order): every tier improves on V7 (e.g. vt34: 34.4%/1.14 vs vt32 32.2%/1.10); 2010-2016 blocks are essentially unchanged.
+- `trend_rider_v10` (same file) — V9 plus a refined profit-take: clear at +28% (not 30%), restore band 8pp (re-enter sooner on dips), 10-day re-clear lockout after a restore (stops the 2022-style clear/restore churn on GLD/DBC). Registered as `trend_rider_v10` (VT34), `_vt32`, `_vt36`. Engine window to 2026-10-07: v10 55.9%/1.66/MaxDD -27.4%, v10_vt32 55.0%/1.68, v10_vt36 56.2%/1.63 — beats every other registered strategy on both CAGR and Sharpe (previous best: v9_vt36 52.5%, v9_vt24 1.58). 2010-2026 live-book simulation confirms robustness: the PT refinement lifts every 3-4y block's Sharpe (2010-13 0.83→0.92, 2016-21 1.74→1.89, 2021-26 1.38→1.43; full period 36.8%/1.22 vs 34.4%/1.14 for plain V9-trail). Research trail: `research/tr9_{lab,run,long,trials,audit}.py`; rejected knobs are logged in `data/research/results/trials_us.jsonl` (macro throttles on VIX term/credit, RSP breadth, instrument MA exits, fast re-entry, SOXL/TECL/TMF/SQQQ baskets).
 - `qqq_trend.py` — `QQQVolTrend`: QQQ exposure = 1{QQQ > 200-day average} × min(1.5, 30% / EWMA vol), held as TQQQ (E/3) + SHY. 1990-2026 on synthetic ETFs: 16.0% vs QQQ 14.9%, MaxDD -37% vs -83%; 2011-2026 on real ETFs it trails QQQ (17.5% vs 19.1%) with a smaller drawdown. `us_momentum_qqq_trend` (in `sleeve_portfolio.py`) holds it half and half with `overnight_recovery_top10`. Research behind it (`research/lab/qqq_scan.py`, `beat_qqq.py`): measured as alpha against QQQ with a 2006-16 design / 2017-26 holdout split, momentum top 10 is the only stock selection whose alpha held (+10%/yr, t 1.4); single factors, walk-forward ML, a mega-cap universe, SEC insider trades, 10-K/10-Q text similarity, earnings-announcement returns, industry momentum and a PIT S&P 400/600 universe (`panel_mid.py`) showed none. In the web engine (2016-09 to 2026-09, $2/order) `us_momentum_qqq_trend` earns 21.0% vs QQQ 20.9%, MaxDD -32%.
+- `svr_momentum.py` — RESEARCH ONLY (not registered): `overnight_recovery_top10` + a FINRA short-volume veto (drop top-70%-quantile SVR names). The 2026-10-07 window looked good (Sharpe 0.87→0.89, MaxDD -47%→-39%) but the quantile neighborhood killed it — 0.60/0.75/0.80 vetoes all score below baseline. A lone spike; registered and reverted the same day. The raw cross-sectional reading (SVR negative predictor, `research/lab/short_volume.py`) stands; the concentrated-portfolio exclusion rule does not.
 - `agent_review.py` — `overnight_recovery_agent_top10`: overnight_recovery's top 50, re-ranked each day by an LLM agent's scores (−2..+2 from filings, fundamentals, demand-chain logic, event risk; `final = quant percentile + 0.25 × score`), vetoes for five event types, top 10 on the base strategy's rebalance rows and hysteresis. Score files in `data/agent/us/` are stamped by `agent_review.py submit` and steer a session only if stamped before its 16:00 New York close; without one the strategy holds exactly what `overnight_recovery_top10` holds, so the two books' difference measures the agent. `AGENT_REVIEW_STRATEGIES` in trader.py keeps it US-only and out of `precompute_backtest.py` (it has no history before its first score file). Research behind the design (2026-09-30): picking 10 of the quant top 50 at random earns 13.4%/yr vs 26.2% for the top 10 (2006-2026), and an LLM cannot be backtested before its training cutoff, so the record starts 2026-09-29.
 - `factor_combo.py` — Configurable factor-combination strategies with daily/weekly/biweekly/monthly rebalancing. US champions: `rec_mfilt+deep_upvol`, `ma200+mom7m+rec126`, `rec_mfilt+ma200`, `mom7m+rec126`. CN champions: `up_cap+quality_mom`, `down_resil+qual_mom`, `rec63+mom_gap`, `up_cap+mom_gap` — these were mined on today's CSI 300 members; on PIT members they earn ~3-7% a year (2016-2026). All registered in trader.py as `fc_{signal}_{freq}` (e.g., `fc_rec_mfilt_deep_upvol_monthly`). 32 new strategies total.

--- a/research/factor_panel_check.py
+++ b/research/factor_panel_check.py
@@ -0,0 +1,73 @@
+"""Pre-vs-post Tiingo overlay factor check for the S&P panel.
+
+Builds the panel twice in-process — Yahoo-only (pre, survivorship-biased)
+and with the Tiingo overlay (post) — and runs the sleeve-relevant factors
+through the standard top-N phase-averaged backtest on both. The delta is the
+survivorship effect on the factors behind the live stock sleeves
+(overnight_recovery: overnight_mom + rec_126; lowrisk: ivol + low_beta).
+
+    PYTHONPATH=. uv run python -m research.factor_panel_check
+"""
+from __future__ import annotations
+
+import os
+import time
+import warnings
+
+import pandas as pd
+
+import research.lab.panel as pnl
+from research.lab.factor_scan import PERIODS, phase_avg
+from research.lab.factors import FACTORS
+from research.lab.report import forward_returns, ic_series, ic_summary
+
+warnings.filterwarnings("ignore")
+
+NAMES = ["overnight_mom", "rec_126", "mom_12_1", "low_vol",
+         "ivol", "low_beta", "sharpe_mom", "amp_cut_mom"]
+RESULTS = os.path.join("data", "research", "results")
+
+
+def scan(p, label: str) -> pd.DataFrame:
+    fwd5 = forward_returns(p, 5)
+    rows = []
+    for name in NAMES:
+        t0 = time.time()
+        s = FACTORS[name](p)
+        row = {"panel": label, "factor": name}
+        for per, (a, b) in PERIODS["us"].items():
+            st = phase_avg(p, s, a, b, 20, 5)
+            ic = ic_summary(ic_series(s.loc[a:b], fwd5.loc[a:b], p.member.loc[a:b], 5))
+            row.update({f"{per}_cagr": st["cagr"], f"{per}_sharpe": st["sharpe"],
+                        f"{per}_ic5": ic["ic"], f"{per}_ic5_t": ic["t"]})
+        rows.append(row)
+        print(f"[{label}] {name:14s} D {row['design_cagr']:6.1%}/{row['design_sharpe']:.2f}  "
+              f"H {row['holdout_cagr']:6.1%}/{row['holdout_sharpe']:.2f}  [{time.time() - t0:.0f}s]",
+              flush=True)
+    return pd.DataFrame(rows)
+
+
+def main() -> None:
+    post = pnl.load_us()
+    orig = pnl.apply_sp500
+    pnl.apply_sp500 = lambda raw, **kw: raw
+    try:
+        pre = pnl._build_us()
+    finally:
+        pnl.apply_sp500 = orig
+    a = scan(pre, "pre")
+    b = scan(post, "post")
+    df = pd.concat([a, b], ignore_index=True)
+    os.makedirs(RESULTS, exist_ok=True)
+    df.to_csv(os.path.join(RESULTS, "factor_panel_check.csv"), index=False)
+    m = a.merge(b, on="factor", suffixes=("_pre", "_post"))
+    print("\nfactor          D_cagr pre->post    D_sh pre->post   H_cagr pre->post    H_sh pre->post")
+    for _, r in m.iterrows():
+        print(f"{r['factor']:14s} {r['design_cagr_pre']:7.1%}->{r['design_cagr_post']:6.1%}   "
+              f"{r['design_sharpe_pre']:5.2f}->{r['design_sharpe_post']:5.2f}   "
+              f"{r['holdout_cagr_pre']:7.1%}->{r['holdout_cagr_post']:6.1%}   "
+              f"{r['holdout_sharpe_pre']:5.2f}->{r['holdout_sharpe_post']:5.2f}")
+
+
+if __name__ == "__main__":
+    main()
--- a/research/lab/fetch_tiingo.py
+++ b/research/lab/fetch_tiingo.py
@@ -0,0 +1,122 @@
+"""Fetch EOD prices from Tiingo (free tier: 500 unique symbols/month, 50/hour).
+
+Used for: (1) completing delisted/missing names in the research panels,
+(2) independent cross-validation of the Yahoo-sourced closes. The key comes
+from TIINGO_API_KEY (environment or .env) and is never logged or written.
+
+Writes data/research/tiingo/<ticker>.csv (resume-safe; a cached symbol never
+counts against the monthly quota again). Columns: as-traded open/high/low/
+close/volume (Tiingo's close is NOT split-adjusted), split/dividend-adjusted
+adjOpen..adjVolume, plus splitFactor and divCash.
+
+    PYTHONPATH=. uv run python -m research.lab.fetch_tiingo TICKER [TICKER ...]
+    PYTHONPATH=. uv run python -m research.lab.fetch_tiingo --file symbols.txt
+    PYTHONPATH=. uv run python -m research.lab.fetch_tiingo --file l.txt --min-interval 75
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+import time
+
+import pandas as pd
+import requests
+
+DIR = os.path.join("data", "research", "tiingo")
+BASE = "https://api.tiingo.com/tiingo/daily/{t}/prices"
+KEEP = ("date", "open", "high", "low", "close", "volume",
+        "adjOpen", "adjHigh", "adjLow", "adjClose", "adjVolume",
+        "splitFactor", "divCash")
+
+
+def api_key() -> str:
+    key = os.environ.get("TIINGO_API_KEY")
+    if not key and os.path.exists(".env"):
+        for line in open(".env"):
+            m = re.match(r"\s*TIINGO_API_KEY\s*=\s*(.+)\s*$", line)
+            if m:
+                key = m.group(1).strip().strip("'\"")
+                break
+    if not key:
+        raise SystemExit("TIINGO_API_KEY not found (env or .env)")
+    return key
+
+
+def fetch(ticker: str, key: str, start: str = "1990-01-01") -> pd.DataFrame | None:
+    """One symbol's full EOD history; None when Tiingo does not have it (404).
+    Network errors back off and retry; a hard failure skips the ticker."""
+    for attempt in range(4):
+        try:
+            r = requests.get(BASE.format(t=ticker.lower()),
+                             params={"startDate": start, "format": "json"},
+                             headers={"Content-Type": "application/json",
+                                      "Authorization": f"Token {key}"}, timeout=60)
+        except requests.RequestException as exc:
+            print(f"{ticker}: {type(exc).__name__}; retry {attempt + 1}", flush=True)
+            time.sleep(20 * (attempt + 1))
+            continue
+        if r.status_code == 429:
+            print(f"{ticker}: rate limited; waiting 15 min", flush=True)
+            time.sleep(900)
+            continue
+        if r.status_code in (402, 403):     # monthly symbol quota exhausted
+            raise SystemExit(f"{ticker}: HTTP {r.status_code} — quota reached; "
+                             "remaining tickers stay uncached for the next run")
+        if r.status_code == 404:
+            return None
+        r.raise_for_status()
+        d = r.json()
+        if not d:
+            return None
+        df = pd.DataFrame(d)
+        df["date"] = pd.to_datetime(df["date"]).dt.tz_localize(None)
+        keep = [c for c in KEEP if c in df.columns]
+        return df[keep].set_index("date")
+    raise RuntimeError(f"{ticker}: network retries exhausted")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("tickers", nargs="*")
+    ap.add_argument("--file", help="one ticker per line")
+    ap.add_argument("--min-interval", type=float, default=75.0,
+                    help="seconds between network fetches (free tier allows 50/hour)")
+    ap.add_argument("--start", default="1990-01-01")
+    args = ap.parse_args()
+    tickers = [l.strip() for l in open(args.file) if l.strip()] if args.file else args.tickers
+    if not tickers:
+        raise SystemExit("usage: fetch_tiingo TICKER ... | --file list.txt")
+
+    os.makedirs(DIR, exist_ok=True)
+    key = api_key()
+    todo = [t for t in tickers if not os.path.exists(os.path.join(DIR, f"{t.upper()}.csv"))]
+    print(f"{len(tickers)} tickers, {len(todo)} to fetch", flush=True)
+    fetched = failed = 0
+    last = 0.0
+    for i, t in enumerate(todo):
+        wait = args.min_interval - (time.time() - last)
+        if wait > 0:
+            time.sleep(wait)
+        last = time.time()
+        try:
+            df = fetch(t, key, args.start)
+        except RuntimeError:
+            print(f"[{i + 1}/{len(todo)}] {t:8s} network fail; retried next run", flush=True)
+            continue
+        if df is None:
+            failed += 1
+            print(f"[{i + 1}/{len(todo)}] {t:8s} not found", flush=True)
+            continue
+        df.to_csv(os.path.join(DIR, f"{t.upper()}.csv"))
+        fetched += 1
+        eta_min = (len(todo) - i - 1) * args.min_interval / 60
+        print(f"[{i + 1}/{len(todo)}] {t:8s} {len(df)} rows "
+              f"{df.index.min().date()}..{df.index.max().date()} (ETA {eta_min:.0f}m)",
+              flush=True)
+    print(f"done: {fetched} fetched, {failed} not found", flush=True)
+
+
+if __name__ == "__main__":
+    main()
--- a/research/lab/fetch_us_mid.py
+++ b/research/lab/fetch_us_mid.py
@@ -56,6 +56,7 @@ import pandas as pd
 ROOT = os.path.join("data", "research", "ext")
 HOLD_DIR = os.path.join(ROOT, "ishares")
 PX_DIR = os.path.join(ROOT, "yahoo_mid")
+TIINGO_DIR = os.path.join(ROOT, "tiingo_mid")
 HOLDINGS = os.path.join(HOLD_DIR, "holdings.csv")
 FUNDS = {"IJH": 239763, "IJR": 239774}          # S&P MidCap 400, S&P SmallCap 600
 ETFS = ["IJH", "IJR", "MDY", "SLY", "IWM", "SPY", "QQQ", "BIL", "SHY"]
@@ -394,10 +395,14 @@ def fetch_prices(start: str = "2005-01-01", retry_missing: bool = False) -> None


 def load_bars(t: str) -> pd.DataFrame | None:
-    """Adjusted OHLCV (as yfinance's auto_adjust=True) plus the unadjusted close."""
+    """Adjusted OHLCV (as yfinance's auto_adjust=True) plus the unadjusted
+    close. Falls back to a Tiingo-converted CSV (see lab.tiingo_merge)."""
    path = _px_path(t)
    if not os.path.exists(path):
-        return None
+        alt = os.path.join(TIINGO_DIR, os.path.basename(path))
+        if not os.path.exists(alt):
+            return None
+        path = alt
    h = pd.read_csv(path, index_col=0, parse_dates=True)
    h = h[~h.index.duplicated(keep="last")].sort_index()
    ratio = (h["Adj Close"] / h["Close"]).where(h["Close"] > 0)
--- a/research/lab/panel.py
+++ b/research/lab/panel.py
@@ -18,6 +18,7 @@ import numpy as np
 import pandas as pd

 from research.lab.engine import Costs, Panel
+from research.lab.tiingo_merge import apply_sp500

 RAW_DIR = os.path.join("data", "research")
 US_ETFS = {"SPY", "RSP", "QQQ", "SHY", "BIL", "IEF", "TLT", "GLD",
@@ -54,6 +55,7 @@ def _us_membership(dates: pd.DatetimeIndex, tickers: list[str]) -> pd.DataFrame:
 def _build_us() -> Panel:
    with open(os.path.join(RAW_DIR, "us_ohlcv.pkl"), "rb") as fh:
        raw = pickle.load(fh)
+    raw = apply_sp500(raw)
    close = raw["close"]
    # keep rows where most listed tickers printed (drops stray holiday rows)
    alive = close.notna().sum(axis=1)
@@ -92,8 +94,31 @@ def _us_sectors(stocks: list[str]) -> pd.Series:
    return sec.reindex(stocks)


+def _tiingo_sig() -> list:
+    d = os.path.join(RAW_DIR, "tiingo")
+    if not os.path.isdir(d):
+        return [0, 0]
+    files = [f for f in os.listdir(d) if f.endswith(".csv")]
+    mtime = max((os.path.getmtime(os.path.join(d, f)) for f in files), default=0)
+    return [len(files), round(mtime, 3)]
+
+
 def load_us() -> Panel:
-    return _cached("us", _build_us)
+    """Cached US panel, rebuilt whenever the Tiingo overlay changes."""
+    path = os.path.join(RAW_DIR, "us.panel.pkl")
+    key = os.path.join(RAW_DIR, "us.panel.key")
+    sig = _tiingo_sig()
+    if os.path.exists(path) and os.path.exists(key):
+        with open(key) as fh:
+            if json.load(fh) == sig:
+                with open(path, "rb") as fh:
+                    return pickle.load(fh)
+    p = _build_us()
+    with open(path, "wb") as fh:
+        pickle.dump(p, fh)
+    with open(key, "w") as fh:
+        json.dump(sig, fh)
+    return p


 # ---------------------------------------------------------------------------
--- a/research/lab/panel_gaps.py
+++ b/research/lab/panel_gaps.py
@@ -0,0 +1,134 @@
+"""Quantify price gaps against PIT membership in the US research panels and
+emit a Tiingo backfill plan (free tier: 500 unique symbols/month, 50/hour).
+
+For S&P 500 ever-members: names with no data, a late start or an early end
+relative to their membership intervals, in both the production books
+(data/us.csv) and the research panel (data/research/us_ohlcv.pkl). For the
+mid/small panels: the Yahoo-missing tickers (data/research/ext/yahoo_mid/
+missing.json) ranked by snapshot weight and recency, since only the top of
+that list fits the monthly quota.
+
+Writes data/research/tiingo_plan.json.
+
+    PYTHONPATH=. uv run python -m research.lab.panel_gaps
+"""
+from __future__ import annotations
+
+import json
+import os
+import pickle
+
+import numpy as np
+import pandas as pd
+
+TOL_DAYS = 45
+PLAN = os.path.join("data", "research", "tiingo_plan.json")
+
+
+def sp500_intervals() -> dict:
+    with open(os.path.join("data", "sp500_history.json")) as fh:
+        return json.load(fh)["intervals"]
+
+
+def classify(px: pd.DataFrame, intervals: dict) -> pd.DataFrame:
+    """Per ever-member: membership vs data presence within the panel window."""
+    rows = []
+    pstart, pend = px.index[0], px.index[-1]
+    for t, iv in intervals.items():
+        iv = [(pd.Timestamp(s) if s else None, pd.Timestamp(e) if e else None) for s, e in iv]
+        mstart = max(min((s for s, _ in iv if s), default=pstart), pstart)
+        mend = min(max((e for _, e in iv if e), default=pend), pend)
+        if mend <= pstart or mstart >= pend:
+            continue
+        col = px[t].dropna() if t in px.columns else pd.Series(dtype=float)
+        first = col.index[0] if len(col) else None
+        last = col.index[-1] if len(col) else None
+        n_mem = int(((px.index >= mstart) & (px.index < mend)).sum())
+        n_px = int(((col.index >= mstart) & (col.index < mend)).sum()) if len(col) else 0
+        flags = []
+        if not len(col):
+            flags.append("no_data")
+        else:
+            if first > mstart + pd.Timedelta(days=TOL_DAYS):
+                flags.append("late_start")
+            if last < mend - pd.Timedelta(days=TOL_DAYS):
+                flags.append("early_end")
+        rows.append(dict(ticker=t, m_first=mstart.date(), m_last=mend.date(),
+                         px_first=first.date() if first is not None else None,
+                         px_last=last.date() if last is not None else None,
+                         n_mem=n_mem, n_px=n_px,
+                         cov=round(n_px / n_mem, 3) if n_mem else np.nan,
+                         flags=",".join(flags)))
+    return pd.DataFrame(rows)
+
+
+def mid_small_plan() -> pd.DataFrame:
+    """Rank the Yahoo-missing mid/small tickers: total snapshot weight, with
+    recent years weighted fully and older ones decayed."""
+    from research.lab.fetch_us_mid import load_holdings
+
+    miss_path = os.path.join("data", "research", "ext", "yahoo_mid", "missing.json")
+    missing = set(json.load(open(miss_path)))
+    h = load_holdings()
+    h = h[h["ticker"].isin(missing)]
+    if h.empty:
+        return h
+    recency = pd.Series(1.0, index=h.index)
+    recency[h["date"] < "2014-01-01"] = 0.25
+    recency[(h["date"] >= "2014-01-01") & (h["date"] < "2019-01-01")] = 0.5
+    h = h.assign(score=h["weight"] * recency)
+    g = h.groupby("ticker")
+    out = pd.DataFrame({
+        "n_snaps": g.size(),
+        "w_sum": g["weight"].sum().round(4),
+        "score": g["score"].sum().round(4),
+        "first": g["date"].min().dt.date,
+        "last": g["date"].max().dt.date,
+        "name": g["name"].first(),
+    }).sort_values("score", ascending=False)
+    return out
+
+
+def main() -> None:
+    intervals = sp500_intervals()
+
+    import data_manager
+    prod = data_manager.load_raw("us")
+    print(f"production data/us.csv: {prod.shape[1]} cols {prod.index[0].date()}..{prod.index[-1].date()}")
+
+    from research.lab.tiingo_merge import apply_sp500
+    with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh:
+        raw = pickle.load(fh)
+    res = apply_sp500(raw, verbose=False)["close"]
+    print(f"research us_ohlcv.pkl + tiingo: {res.shape[1]} cols "
+          f"{res.index[0].date()}..{res.index[-1].date()}")
+
+    plan: dict[str, list[str]] = {}
+    for label, px in (("production", prod), ("research", res)):
+        df = classify(px, intervals)
+        bad = df[df["flags"] != ""].sort_values(["n_mem"], ascending=False)
+        print(f"\n[{label}] ever-members with intervals in window: {len(df)}; with flags: {len(bad)}")
+        for fl in ("no_data", "late_start", "early_end"):
+            sub = bad[bad["flags"].str.contains(fl)]
+            print(f"  {fl}: {len(sub)}")
+            cols = ["ticker", "m_first", "m_last", "px_first", "px_last", "cov"]
+            if len(sub):
+                print(sub[cols].head(25).to_string(index=False))
+        names = sorted(set(bad[bad["flags"].str.contains("no_data|early_end")]["ticker"]))
+        plan[f"sp500_{label}"] = names
+        print(f"  -> {len(names)} no_data/early_end names")
+
+    ms = mid_small_plan()
+    print(f"\n[mid/small] missing tickers with snapshot weight: {len(ms)}")
+    print(ms.head(30).to_string())
+    print(f"  weight coverage reachable in quota (top 500): "
+          f"{ms['w_sum'].head(500).sum() / ms['w_sum'].sum():.1%}")
+    plan["mid_small"] = list(ms.index)
+
+    with open(PLAN, "w") as fh:
+        json.dump(plan, fh, indent=1)
+    print(f"\nSaved {PLAN}")
+
+
+if __name__ == "__main__":
+    main()
--- a/research/lab/tiingo_merge.py
+++ b/research/lab/tiingo_merge.py
@@ -0,0 +1,331 @@
+"""Overlay Tiingo EOD caches (data/research/tiingo/) onto the US research panels.
+
+S&P 500 -- ``apply_sp500(raw)``: fills the Yahoo-built panel's gaps, whole
+missing columns for delisted names and late tails, keeping Yahoo cells where
+both sources have data. Safety per ticker: >=50% of its PIT membership
+sessions must be covered; where both sources overlap (>=20 sessions) the
+median Tiingo/Yahoo close ratio must sit within 2% (same total-return
+adjustment basis; a recycled ticker fails this). Two rename splices
+(GPS<-GAP, DISCA<-WBD) backfill pre-rename eras from the successor ticker.
+Read-only on the Yahoo artifact: deleting data/research/tiingo/ reverts it.
+The merge prints an audit report incl. terminal gaps (last bar vs membership
+end), which is the delisted-terminal check.
+
+Mid/small -- ``write_yahoo_mid()``: converts the caches to the yahoo_mid CSV
+format (split-adjusted OHLC with a Stock Splits column so load_bars can
+recover the as-traded close, Adj Close as served) under
+data/research/ext/tiingo_mid/, which fetch_us_mid.load_bars falls back to.
+
+    PYTHONPATH=. uv run python -m research.lab.tiingo_merge --check
+    PYTHONPATH=. uv run python -m research.lab.tiingo_merge --yahoo-mid
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+import numpy as np
+import pandas as pd
+
+DIR = os.path.join("data", "research", "tiingo")
+MID_DIR = os.path.join("data", "research", "ext", "tiingo_mid")
+FIELDS = ("open", "high", "low", "close", "volume")
+SPLICE = {"GPS": "GAP", "DISCA": "WBD"}
+MIN_COVERAGE = 0.5
+MIN_OVERLAP = 20
+RATIO_TOL = 0.02
+TERMINAL_GAP_DAYS = 45
+# Pure ticker changes where the successor's series is the same security's
+# continuous price history (Yahoo/Tiingo migrate it on rename). NOT for
+# merger-shell cases (RTN->RTX pre-2020 is United Technologies, not Raytheon).
+SPLICE.update({"WLTW": "WTW", "ADS": "BFH", "CCE": "CCEP",
+               "ESV": "VAL", "FBHS": "FBIN", "HFC": "DINO"})
+
+_intervals_cache: dict | None = None
+
+
+def _intervals() -> dict:
+    global _intervals_cache
+    if _intervals_cache is None:
+        with open(os.path.join("data", "sp500_history.json")) as fh:
+            _intervals_cache = json.load(fh)["intervals"]
+    return _intervals_cache
+
+
+def _cached(t: str) -> pd.DataFrame:
+    h = pd.read_csv(os.path.join(DIR, f"{t.upper()}.csv"), index_col=0, parse_dates=True)
+    return h[~h.index.duplicated(keep="last")].sort_index()
+
+
+def cached_tickers() -> list[str]:
+    if not os.path.isdir(DIR):
+        return []
+    return sorted(f[:-4] for f in os.listdir(DIR) if f.endswith(".csv"))
+
+
+def _frame(t: str) -> pd.DataFrame:
+    """Adjusted OHLCV in the panel convention (auto_adjust=True)."""
+    h = _cached(t)
+    out = pd.DataFrame({f: h["adj" + f[0].upper() + f[1:]] for f in FIELDS})
+    return out
+
+
+def _member_mask(t: str, dates: pd.DatetimeIndex) -> np.ndarray:
+    m = np.zeros(len(dates), bool)
+    for s, e in _intervals().get(t, []):
+        si = dates.searchsorted(pd.Timestamp(s) if s else dates[0], side="left")
+        ei = dates.searchsorted(pd.Timestamp(e) if e else dates[-1] + pd.Timedelta(days=1),
+                                side="left")
+        m[si:ei] = True
+    return m
+
+
+def _coverage(t: str, frame: pd.DataFrame, dates: pd.DatetimeIndex) -> float:
+    mask = _member_mask(t, dates)
+    if not mask.any():
+        return 0.0
+    have = frame["close"].reindex(dates).notna().to_numpy()
+    return float((mask & have).sum()) / float(mask.sum())
+
+
+def _cross_check(yahoo: pd.Series, tiingo: pd.Series) -> float | None:
+    """Median Tiingo/Yahoo close ratio over the overlap; None if too small."""
+    y = yahoo.reindex(tiingo.index)
+    both = y.notna() & tiingo["close"].notna()
+    if both.sum() < MIN_OVERLAP:
+        return None
+    return float((tiingo.loc[both, "close"] / y[both]).median())
+
+
+def apply_sp500(raw: dict, *, verbose: bool = True) -> dict:
+    """Return a copy of the raw panel dict with Tiingo cells filled in."""
+    dates = raw["close"].index
+    out = {k: v.copy() for k, v in raw.items()}
+    merged, extended, replaced, skipped, rejected, spliced, terminals = [], [], [], [], [], [], []
+    frames = {t: _frame(t) for t in cached_tickers()}
+
+    def fill_column(t: str, frame: pd.DataFrame, mask: np.ndarray | None = None):
+        added = t not in out["close"].columns
+        for f in FIELDS:
+            col = frame[f].reindex(dates)
+            if mask is not None:
+                col = col.where(mask)
+            if added:
+                out[f][t] = col
+            else:
+                out[f][t] = out[f][t].where(out[f][t].notna(), col)
+        return added
+
+    for t, frame in frames.items():
+        cov = _coverage(t, frame, dates)
+        if cov < MIN_COVERAGE:
+            skipped.append((t, round(cov, 3)))
+            continue
+        replace = False
+        if t in out["close"].columns:
+            mask = _member_mask(t, dates)
+            have = out["close"][t].reindex(dates).notna().to_numpy()
+            if mask.any() and (mask & have).sum() / mask.sum() < MIN_COVERAGE:
+                # a recycled ticker or a broken stub sits in the column: the
+                # full series wins wherever it has data
+                replace = True
+            else:
+                med = _cross_check(out["close"][t], frame)
+                if med is not None and not (1 - RATIO_TOL < med < 1 + RATIO_TOL):
+                    rejected.append((t, round(med, 4)))
+                    continue
+        if replace:
+            for f in FIELDS:
+                col = frame[f].reindex(dates)
+                out[f][t] = col.where(col.notna(), out[f][t])
+            replaced.append(t)
+        else:
+            added = fill_column(t, frame)
+            (merged if added else extended).append(t)
+        # terminal check: last bar vs membership end within the panel window
+        col = out["close"][t].dropna()
+        ends = [pd.Timestamp(e) if e else dates[-1] + pd.Timedelta(days=1)
+                for _, e in _intervals().get(t, [])]
+        mend = min(max(ends), dates[-1]) if ends else None
+        if mend is not None and len(col):
+            gap = (mend - col.index[-1]).days
+            if gap > TERMINAL_GAP_DAYS:
+                terminals.append((t, str(col.index[-1].date()), str(mend.date()), gap))
+
+    for old, new in SPLICE.items():
+        frame = frames.get(new)
+        if frame is None and new in out["close"].columns:
+            frame = pd.DataFrame({f: out[f][new] for f in FIELDS})
+        if frame is None:
+            continue
+        mask = _member_mask(old, dates)
+        if not mask.any():
+            continue
+        have = out["close"][old].reindex(dates).notna().to_numpy() if old in out["close"] else np.zeros(len(dates), bool)
+        if (mask & have).sum() / mask.sum() >= MIN_COVERAGE:
+            continue
+        fill_column(old, frame, mask)
+        spliced.append(f"{old}<-{new}")
+
+    if verbose:
+        print(f"tiingo overlay: {len(frames)} cached files; merged {len(merged)}, "
+              f"extended {len(extended)}, replaced {len(replaced)}, spliced {len(spliced)}, "
+              f"skipped {len(skipped)}, rejected {len(rejected)}")
+        if merged:
+            print(f"  merged: {', '.join(merged)}")
+        if extended:
+            print(f"  extended: {', '.join(extended)}")
+        if replaced:
+            print(f"  replaced (column was a stub/recycled): {', '.join(replaced)}")
+        if spliced:
+            print(f"  spliced: {', '.join(spliced)}")
+        if skipped:
+            print(f"  skipped (coverage<{MIN_COVERAGE}): "
+                  + ", ".join(f"{t}:{c}" for t, c in skipped[:30])
+                  + (" ..." if len(skipped) > 30 else ""))
+        if rejected:
+            print(f"  rejected (ratio off >{RATIO_TOL:.0%}): "
+                  + ", ".join(f"{t}:{r}" for t, r in rejected))
+        if terminals:
+            print(f"  terminal gaps >{TERMINAL_GAP_DAYS}d: "
+                  + ", ".join(f"{t} last={d} member_end={m} ({g}d)" for t, d, m, g in terminals))
+    return out
+
+
+PROD_FIELDS = ("open", "high", "low", "volume")
+
+
+def _production_plan() -> tuple:
+    """Which cached tickers to merge into the production store, and how."""
+    import data_manager
+    base = data_manager.load_raw("us")
+    if base is None:
+        raise FileNotFoundError("data/us.csv not found")
+    dates = base.index
+    frames = {t: _frame(t) for t in cached_tickers()}
+    add, extend, replace, skip, rejected = [], [], [], [], []
+    for t, fr in frames.items():
+        cov = _coverage(t, fr, dates)
+        if cov < MIN_COVERAGE:
+            skip.append((t, round(cov, 3)))
+            continue
+        if t in base.columns:
+            mask = _member_mask(t, dates)
+            have = base[t].reindex(dates).notna().to_numpy()
+            if mask.any() and (mask & have).sum() / mask.sum() < MIN_COVERAGE:
+                replace.append(t)
+                continue
+            med = _cross_check(base[t], fr)
+            if med is not None and not (1 - RATIO_TOL < med < 1 + RATIO_TOL):
+                rejected.append((t, round(med, 4)))
+                continue
+            n = int(base[t].notna().sum())
+            (extend if n else add).append(t)
+        else:
+            add.append(t)
+    for old, new in SPLICE.items():
+        if old in add:
+            continue
+        source = frames.get(new)
+        if source is None and new in base.columns:
+            source = pd.DataFrame({f: data_manager.load_raw("us", f)[new]
+                                   for f in ("close",) + PROD_FIELDS})
+        if source is None:
+            continue
+        mask = _member_mask(old, dates)
+        if not mask.any():
+            continue
+        have = (base[old].reindex(dates).notna().to_numpy() if old in base.columns
+                else np.zeros(len(dates), bool))
+        if (mask & have).sum() / mask.sum() < MIN_COVERAGE:
+            add.append(old)
+            fr = source.reindex(dates)
+            frames[old] = fr.where(np.broadcast_to(mask[:, None], fr.shape))
+    return base, frames, add, extend, replace, skip, rejected
+
+
+def apply_production(*, dry_run: bool = False) -> None:
+    """Merge Tiingo frames into data/us*.csv (backup .bak-tiingo on first write)."""
+    import data_manager
+    from data_manager import _data_path, _panel_lock, _write_atomic
+    base, frames, add, extend, replace, skip, rejected = _production_plan()
+    names = add + extend + replace
+    print(f"production merge: {len(frames)} cached; add {len(add)}, extend {len(extend)}, "
+          f"replace {len(replace)}, skip {len(skip)}, rejected {len(rejected)}")
+    if add:
+        print(f"  add: {', '.join(add)}")
+    if extend:
+        print(f"  extend: {', '.join(extend)}")
+    if replace:
+        print(f"  replace: {', '.join(replace)}")
+    if rejected:
+        print(f"  rejected: {', '.join(f'{t}:{r}' for t, r in rejected)}")
+    if dry_run or not names:
+        return
+    with _panel_lock("us"):
+        fields = {f: data_manager.load_raw("us", f) for f in PROD_FIELDS}
+        repl = set(replace)
+        for f in FIELDS:
+            panel = base if f == "close" else fields[f]
+            if panel is None:
+                continue
+            for t in names:
+                col = frames[t][f].reindex(panel.index)
+                if t not in panel.columns:
+                    panel[t] = col
+                elif t in repl:
+                    panel[t] = col.where(col.notna(), panel[t])
+                else:
+                    panel[t] = panel[t].where(panel[t].notna(), col)
+            path = _data_path("us", f)
+            if not os.path.exists(path + ".bak-tiingo"):
+                import shutil
+                shutil.copy2(path, path + ".bak-tiingo")
+            _write_atomic(panel, path)
+        print(f"production store updated for {len(names)} tickers "
+              f"(backups: data/us*.csv.bak-tiingo)")
+
+
+def write_yahoo_mid(tickers: list[str] | None = None) -> None:
+    """Convert caches to yahoo_mid-format CSVs (split-adjusted OHLC + Stock
+    Splits + as-traded close recoverable via load_bars)."""
+    os.makedirs(MID_DIR, exist_ok=True)
+    names = tickers if tickers is not None else cached_tickers()
+    written = 0
+    for t in names:
+        h = _cached(t)
+        if not {"close", "adjClose", "splitFactor"}.issubset(h.columns):
+            print(f"{t}: missing columns; refetch with the current fetch_tiingo")
+            continue
+        splits = h["splitFactor"].replace(0, 1.0).fillna(1.0)
+        later = splits[::-1].cumprod()[::-1].shift(-1).fillna(1.0)
+        out = pd.DataFrame({
+            "Open": h["open"] / later, "High": h["high"] / later,
+            "Low": h["low"] / later, "Close": h["close"] / later,
+            "Adj Close": h["adjClose"], "Volume": h["adjVolume"],
+            "Dividends": h.get("divCash", pd.Series(0.0, index=h.index)).fillna(0.0),
+            "Stock Splits": splits,
+        })
+        out.index.name = "Date"
+        out.to_csv(os.path.join(MID_DIR, f"{t.upper()}.csv"))
+        written += 1
+    print(f"wrote {written} converted CSVs to {MID_DIR}")
+
+
+def main() -> None:
+    if "--yahoo-mid" in sys.argv:
+        args = [a for a in sys.argv[1:] if not a.startswith("--")]
+        write_yahoo_mid(args or None)
+        return
+    if "--production" in sys.argv:
+        apply_production(dry_run="--apply" not in sys.argv)
+        return
+    import pickle
+    with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh:
+        raw = pickle.load(fh)
+    apply_sp500(raw)
+
+
+if __name__ == "__main__":
+    main()
--- a/research/midsmall_rebuild.py
+++ b/research/midsmall_rebuild.py
@@ -0,0 +1,50 @@
+"""Convert mid/small Tiingo caches, rebuild the PIT panels, report the
+survivorship change (pre-Tiingo backup vs rebuilt member counts per year).
+
+    PYTHONPATH=. uv run python -m research.midsmall_rebuild
+"""
+from __future__ import annotations
+
+import os
+import pickle
+import shutil
+
+import pandas as pd
+
+
+def main() -> None:
+    stage = [l.strip() for l in open("data/research/tiingo_midsmall.txt")][:230]
+    cached = [n for n in stage if os.path.exists(f"data/research/tiingo/{n}.csv")]
+    print(f"mid/small tickers: {len(stage)} listed, {len(cached)} cached", flush=True)
+
+    from research.lab.tiingo_merge import write_yahoo_mid
+    write_yahoo_mid(cached)
+
+    for f in ("us_mid_bars.pkl", "us_mid.panel.pkl", "us_small.panel.pkl", "us_midsmall.panel.pkl"):
+        p = os.path.join("data", "research", f)
+        if os.path.exists(p):
+            os.remove(p)
+            print(f"removed {p}", flush=True)
+
+    from research.lab.panel_mid import load_us_mid
+    new = load_us_mid("midsmall")
+    with open("data/research/us_midsmall.panel.pre_tiingo.pkl", "rb") as fh:
+        old = pickle.load(fh)
+
+    n, o = new.member.sum(axis=1), old.member.sum(axis=1)
+    comp = pd.DataFrame({"post": n.resample("YE").last(), "pre": o.resample("YE").last()}).dropna()
+    comp["delta"] = comp["post"] - comp["pre"]
+    comp.index = comp.index.year
+    print("\nPIT members per year (pre -> post):")
+    print(comp.to_string())
+    from research.lab.fetch_us_mid import load_bars
+    restored = [t for t in cached if t in new.close.columns
+                and new.member[t].sum() > 0 and load_bars(t) is not None
+                and t not in old.close.columns]
+    print(f"\nnew PIT-covered names not in old panel: {len(restored)}")
+    print(new.close.shape, flush=True)
+    print("REBUILD DONE", flush=True)
+
+
+if __name__ == "__main__":
+    main()
--- a/research/midsmall_scan.py
+++ b/research/midsmall_scan.py
@@ -0,0 +1,131 @@
+"""Mid/small factor scan with a survivorship experiment.
+
+Compact top-N next-open backtests (full phase averaging, 20bp/side) on the
+PIT S&P 400/600 union Panel, run against BOTH builds: the Yahoo-only panel
+(pre-Tiingo backup, delisted names silently missing) and the Tiingo-extended
+rebuild. A factor whose conclusion flips between the two was a survivorship
+artifact; one that survives both is actionable.
+
+Windows: design 2007-01..2016-12, holdout 2017-01..2026-09. Scores are
+computed on the full history and the index is masked to PIT members, so
+pre-admission bars inform indicators but never selection.
+
+The overnight composite approximates the production robust_momentum sleeve
+(overnight-return momentum + distance from the 126d low), not its exact code.
+
+    PYTHONPATH=. uv run python -m research.midsmall_scan --panel both
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import pickle
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from research.lab.engine import simulate
+from research.lab.report import forward_returns, ic_series, ic_summary, stats
+
+warnings.filterwarnings("ignore")
+
+DESIGN = ("2007-01-01", "2016-12-31")
+HOLDOUT = ("2017-01-01", "2026-09-24")
+RESULTS = os.path.join("data", "research", "results")
+
+
+def load(panel: str):
+    if panel == "new":
+        from research.lab.panel_mid import load_us_mid
+        return load_us_mid("midsmall")
+    with open(os.path.join("data", "research", "us_midsmall.panel.pre_tiingo.pkl"), "rb") as fh:
+        return pickle.load(fh)
+
+
+def factors(p) -> dict:
+    c = p.close.where(p.member)
+    o = p.open.where(p.member)
+    vol = c.pct_change(fill_method=None).rolling(60, min_periods=40).std()
+    onr = ((o / c.shift(1) - 1).where(lambda x: x.abs() < 0.5)
+           .rolling(63, min_periods=40).mean())
+    dist126 = c / c.rolling(126, min_periods=100).min() - 1
+    hi252 = c / c.rolling(252, min_periods=200).max()
+    pr = lambda x: x.rank(axis=1, pct=True)
+    return {
+        "mom_12_1": c.shift(21) / c.shift(252) - 1,
+        "mom_6_1": c.shift(21) / c.shift(126) - 1,
+        "rev_1m": -(c / c.shift(21) - 1),
+        "rev_1w": -(c / c.shift(5) - 1),
+        "lowvol": -vol,
+        "hi252": hi252,
+        "dist126": dist126,
+        "onr63": onr,
+        "orm_comp": pr(onr) + pr(dist126),
+    }
+
+
+def phase_avg(p, s, window, top_n, every) -> dict:
+    a, b = window
+    w0 = p.dates.searchsorted(pd.Timestamp(a))
+    sp = p.slice(p.dates[max(w0 - 5, 0)], b)
+    rows = []
+    for off in range(every):
+        r = simulate(sp, s, top_n=top_n, every=every, offset=off, warmup=5)
+        rows.append(stats(r.equity.loc[a:b], r.turnover))
+    return pd.DataFrame(rows).mean().to_dict()
+
+
+def run(panel: str, top_n: int, every: int) -> pd.DataFrame:
+    p = load(panel)
+    fwd5 = forward_returns(p, 5)
+    rows = []
+    for name, s in factors(p).items():
+        row = {"panel": panel, "factor": name, "top_n": top_n, "every": every}
+        for wname, window in (("design", DESIGN), ("holdout", HOLDOUT)):
+            st = phase_avg(p, s, window, top_n, every)
+            ic = ic_summary(ic_series(s.loc[window[0]:window[1]], fwd5.loc[window[0]:window[1]],
+                                      p.member.loc[window[0]:window[1]], 5))
+            row.update({f"{wname}_cagr": st["cagr"], f"{wname}_sharpe": st["sharpe"],
+                        f"{wname}_mdd": st["mdd"], f"{wname}_turn": st["turnover"],
+                        f"{wname}_ic5": ic["ic"], f"{wname}_ic_t": ic["t"]})
+        rows.append(row)
+        print(f"{name:10s} D cagr {row['design_cagr']:6.1%} sh {row['design_sharpe']:5.2f} "
+              f"ic {row['design_ic5']:+.3f} | H cagr {row['holdout_cagr']:6.1%} "
+              f"sh {row['holdout_sharpe']:5.2f} ic {row['holdout_ic5']:+.3f}", flush=True)
+    return pd.DataFrame(rows)
+
+
+def ew_universe(p) -> None:
+    r = p.close.pct_change(fill_method=None).where(p.member.shift(1, fill_value=False))
+    ew = (1 + r.mean(axis=1).fillna(0.0)).cumprod()
+    b = p.benchmark
+    for wname, (a, bnd) in (("design", DESIGN), ("holdout", HOLDOUT)):
+        se = stats(ew.loc[a:bnd])
+        sb = stats(b.loc[a:bnd])
+        print(f"  EW members {wname}: cagr {se['cagr']:6.1%} sh {se['sharpe']:5.2f} mdd {se['mdd']:6.1%}"
+              f"   |  ETF blend: cagr {sb['cagr']:6.1%} sh {sb['sharpe']:5.2f}")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--panel", default="both", choices=["old", "new", "both"])
+    ap.add_argument("--top-n", type=int, default=20)
+    ap.add_argument("--every", type=int, default=10)
+    args = ap.parse_args()
+    panels = ["old", "new"] if args.panel == "both" else [args.panel]
+    out = []
+    for panel in panels:
+        print(f"\n=== {panel} panel ({args.top_n} names, every {args.every}) ===")
+        p = load(panel)
+        ew_universe(p)
+        out.append(run(panel, args.top_n, args.every))
+    df = pd.concat(out, ignore_index=True)
+    os.makedirs(RESULTS, exist_ok=True)
+    path = os.path.join(RESULTS, f"midsmall_scan_{args.every}.csv")
+    df.to_csv(path, index=False)
+    print(f"saved {path}")
+
+
+if __name__ == "__main__":
+    main()
--- a/research/prod_impact.py
+++ b/research/prod_impact.py
@@ -0,0 +1,75 @@
+"""A/B the Tiingo backfill's effect on registered replays.
+
+Runs the requested strategies through backtest_runner on the current
+production store, then again with the pre-backfill snapshot (data/us*.csv
+.bak-tiingo) swapped in, and prints CAGR / Sharpe / MaxDD deltas. Files are
+swapped under the data_manager panel lock and restored afterwards.
+
+    PYTHONPATH=. uv run python -m research.prod_impact [STRATEGY ...]
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+
+import metrics
+import backtest_runner as br
+import data_manager
+
+FILES = ["us.csv", "us_open.csv", "us_high.csv", "us_low.csv", "us_volume.csv"]
+START = "2018-01-01"
+
+
+def run(names: list[str]) -> dict:
+    r = br.run_market("us", names, None, 10_000.0, 0.001, 2.0, refresh=False, start=START)
+    out = {}
+    for s in r["strategy_runs"]:
+        eq = s["equity"]
+        out[s["name"]] = {
+            "cagr": metrics.annualized_return(eq),
+            "sharpe": metrics.sharpe_ratio(eq.pct_change().dropna()),
+            "mdd": metrics.max_drawdown(eq),
+        }
+    return out
+
+
+def swap(pre_backfill: bool) -> None:
+    """Swap the .bak-tiingo snapshot in or out (live files preserved as .merged-tmp)."""
+    for f in FILES:
+        live = os.path.join("data", f)
+        bak = live + ".bak-tiingo"
+        tmp = live + ".merged-tmp"
+        if not os.path.exists(bak) and not os.path.exists(tmp):
+            continue
+        if pre_backfill:
+            if os.path.exists(bak):
+                shutil.copy2(live, tmp)
+                shutil.copy2(bak, live)
+        else:
+            if os.path.exists(tmp):
+                shutil.copy2(tmp, live)
+                os.remove(tmp)
+
+
+def main() -> None:
+    names = sys.argv[1:] or ["overnight_recovery_top10", "lowrisk_top20"]
+    with data_manager._panel_lock("us"):
+        print("=== merged (Tiingo backfill) ===")
+        a = run(names)
+        swap(pre_backfill=True)
+    try:
+        print("=== pre-backfill snapshot ===")
+        b = run(names)
+    finally:
+        with data_manager._panel_lock("us"):
+            swap(pre_backfill=False)
+    print("\nstrategy                              cagr%          sharpe        mdd%")
+    for n in a:
+        print(f"{n:34s} {a[n]['cagr']*100:6.1f}->{b[n]['cagr']*100:6.1f}  "
+              f"{a[n]['sharpe']:5.2f}->{b[n]['sharpe']:5.2f}  "
+              f"{a[n]['mdd']*100:6.1f}->{b[n]['mdd']*100:6.1f}")
+
+
+if __name__ == "__main__":
+    main()