quant-trading/research/lab/svr_combo.py

"""
Does the low short-volume-ratio factor (low_svr21, the only class-A row of
research/lab/factor_census.py) improve the momentum strategy or the book?

    PYTHONPATH=. uv run --with numba python -m research.lab.svr_combo

low_svr21 = -(21-day mean of FINRA REG SHO ShortVolume / TotalVolume),
shifted 2 sessions (the day-D file is published after D's close).

Arms (research engine: PIT S&P 500, next-open fills, 10 bp a side, phase-averaged):
  base       overnight_mom + rec_126 ranks, top 10, every 10 days, buffer 2
             (the live overnight_recovery_top10)
  blend_w    (1 - w) x base score + w x low_svr21 rank, same construction;
             w = 1/3 gated, w = 0.25 and 0.5 are the neighbourhood. A member
             without FINRA data gets a neutral 0.5 SVR rank, so the universe
             matches base
  svr_sleeve low_svr21 top 20, every 21 days, buffer 2 (the census construction)
Books (overnight_book band_blend: equal shares, 25% band, month-start reset):
  book3      overnight + lowrisk + trend_v7 (three_sleeve_overnight, the paper test)
  book4      book3 + svr_sleeve at 1/4 each                                gated
  reported   book3 with svr_sleeve in place of lowrisk; base + svr_sleeve 50/50

Window: 2018-10-01 .. 2026-09-24, where SVR covers the members. Halves are
split at 2022-07-01. The SVR S&P 500 scan, a veto overlay on the momentum
sleeve (research/svr_overlay.py, reverted for a neighbourhood failure) and
the census all saw this window before this rule was written. low_svr21 was
also picked as the best of 86 census rows. This is a consistency check;
only a forward test can be independent.

The rule is written to prereg.json before the run; the script refuses to
overwrite results.json.

Result (2026-10-12): STOP. Blend: G2 passes; G1 and G3 fail. Sleeve: S1
and S2 fail.

  arm / book (2018-10..2026-09)  CAGR   xSh   halves       MaxDD   alpha vs EW (t)
  base (top 10 momentum)         27.3%  0.81  0.52 / 1.05  -34.9%  +16.4% (1.99)
  blend w=0.25                   28.2%  0.87  0.62 / 1.08  -41.1%  +16.2% (2.24)
  blend w=1/3                    27.1%  0.87  0.67 / 1.05  -41.1%  +15.0% (2.31)
  blend w=0.5                    23.5%  0.82  0.73 / 0.90  -40.9%  +11.8% (2.13)
  svr_sleeve (top 20)            17.5%  0.80  0.58 / 1.16  -38.7%   +5.8% (2.87)
  book3 (paper-test book)        25.1%  1.49  1.84 / 1.19  -15.0%
  book4 (+ svr sleeve)           23.2%  1.42  1.49 / 1.34  -21.6%
  svr in place of lowrisk        28.8%  1.47  1.55 / 1.39  -22.6%
  base + svr sleeve 50/50        23.0%  0.88  0.56 / 1.24  -35.5%

  - The blend helps everywhere a little: Sharpe +0.06 at w=1/3, vol 33% ->
    30%, and better in 2018-24. It gives up most of 2026 (+109% -> +66%) and
    deepens MaxDD by 6pp. That misses the +0.10 bar and the 3pp drawdown bar.
  - The SVR sleeve correlates 0.50 monthly with the momentum sleeve. Lowrisk
    and trend already diversify the book, so a fourth sleeve dilutes it
    (1.49 -> 1.42) and adds 6.6pp of drawdown.
  - Boundary: 8 years, one regime, and the factor was picked as the best of
    86 on this same window.
"""

from __future__ import annotations

import json
import os

import numpy as np
import pandas as pd

from research.lab.alpha import alpha_stats
from research.lab.factor_scan import RESULTS, load
from research.lab.factors import FACTORS
from research.lab.levered_book import stats, tbill
from research.lab.managed_futures_book import band_blend
from research.lab.short_volume import factors as svr_factors, load_svr
from research.lab.us_sweep import run

OUT = os.path.join(RESULTS, "svr_combo")
WIN = ("2018-10-01", "2026-09-24")
HALVES = [("2018-10-01", "2022-06-30"), ("2022-07-01", "2026-09-24")]
STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl")
OVERNIGHT = os.path.join(RESULTS, "overnight_only", "daily.pkl")
W_GATED, W_NEIGHBOURS = 1 / 3, (0.25, 0.5)

PREREG = {
    "question": "Does low_svr21 raise the Sharpe of overnight_recovery_top10 (score blend) or of the "
                "overnight three-sleeve book (extra sleeve) in both halves of 2018-10..2026-09?",
    "window": WIN, "halves": HALVES,
    "gates": {
        "blend (all of)": [
            "G1 blend w=1/3 excess Sharpe >= base + 0.10 over the window and >= base in each half",
            "G2 blend w=0.25 and w=0.5 excess Sharpe >= base over the window (no lone spike)",
            "G3 blend w=1/3 MaxDD >= base MaxDD - 3pp",
        ],
        "sleeve (all of)": [
            "S1 book4 excess Sharpe >= book3 + 0.05 in each half",
            "S2 book4 MaxDD >= book3 MaxDD - 3pp",
        ],
    },
    "decision": {
        "CONTINUE": "an arm passes -> add the daily FINRA short-volume fetch to the live pipeline and run "
                    "the arm forward-only (shadow book) before any live capital",
        "STOP": "no arm passes -> low_svr21 stays a census row",
    },
    "data_status": "window inspected before (SVR scan, veto overlay, census); factor chosen as best of 86",
}


def pct(df: pd.DataFrame, member: pd.DataFrame) -> pd.DataFrame:
    return df.where(member).rank(axis=1, pct=True)


def summary(r: pd.Series, rf: pd.Series, ew: pd.Series) -> dict:
    r = r.loc[WIN[0]:WIN[1]]
    s = stats(r, rf)
    a = alpha_stats(r, ew.reindex(r.index), rf.reindex(r.index))
    return {"cagr": float(s["cagr"]), "sharpe": float(s["sharpe"]), "vol": float(s["vol"]), "mdd": float(s["mdd"]),
            "worst_yr": float(s["worst_yr"]), "alpha_ew": float(a["alpha"]), "alpha_ew_t": float(a["t"]),
            "halves": [float(stats(r.loc[x:y], rf)["sharpe"]) for x, y in HALVES]}


def fmt(k: str, d: dict) -> str:
    return (f"{k:34s} cagr {d['cagr']*100:5.1f}% xsh {d['sharpe']:5.2f} vol {d['vol']*100:4.1f}% "
            f"mdd {d['mdd']*100:6.1f}% alpha/EW {d['alpha_ew']*100:+5.1f}% (t {d['alpha_ew_t']:+.2f}) "
            f"halves {d['halves'][0]:5.2f} {d['halves'][1]:5.2f}")


def main() -> None:
    os.makedirs(OUT, exist_ok=True)
    res_path = os.path.join(OUT, "results.json")
    if os.path.exists(res_path):
        raise SystemExit(f"{res_path} exists; this study does not overwrite results")
    with open(os.path.join(OUT, "prereg.json"), "w") as fh:
        json.dump(PREREG, fh, indent=2)

    p, _ = load("us")
    m = p.member
    house = (pct(FACTORS["overnight_mom"](p), m) + pct(FACTORS["rec_126"](p), m)) / 2
    svr = -svr_factors(load_svr(p.close))["svr21"].shift(2).reindex(index=p.dates, columns=p.close.columns)
    svr_r = pct(svr, m).fillna(0.5).where(m)        # a member without FINRA data is neutral, not dropped
    hr = pct(house, m)
    sp = p.slice("2018-06-01", WIN[1])
    rf = tbill(p.dates).fillna(0.0)
    ret = p.close.pct_change(fill_method=None)
    ew = ret.where(m.shift(1, fill_value=False)).mean(axis=1)
    live = dict(top_n=10, every=10, buffer=2.0)
    subs = [WIN]

    S = {"base": run(sp, house.where(m), subs, **live)["_rets"]}
    for w in (W_NEIGHBOURS[0], W_GATED, W_NEIGHBOURS[1]):
        S[f"blend_w{w:.2f}"] = run(sp, ((1 - w) * hr + w * svr_r).where(m), subs, **live)["_rets"]
    S["svr_sleeve"] = run(sp, svr.where(m), subs, top_n=20, every=21, buffer=2.0)["_rets"]
    S = pd.DataFrame(S).loc[WIN[0]:WIN[1]]

    R = pd.read_pickle(STREAMS)
    O = pd.read_pickle(OVERNIGHT)
    B = pd.DataFrame({"overnight": O["sleeve_score"], "lowrisk": R["lowrisk"], "trend_v7": R["trend_v7"]})
    B = B.join(S[["svr_sleeve", "base"]], how="inner").loc[WIN[0]:WIN[1]]
    books = {
        "book3": band_blend(B[["overnight", "lowrisk", "trend_v7"]], rf)[0],
        "book4": band_blend(B[["overnight", "lowrisk", "trend_v7", "svr_sleeve"]], rf)[0],
        "book3_svr_for_lowrisk": band_blend(B[["overnight", "svr_sleeve", "trend_v7"]], rf)[0],
        "base+svr_sleeve 50/50": band_blend(B[["base", "svr_sleeve"]], rf)[0],
    }
    res = {"arms": {}, "books": {}}
    print("=== arms")
    for k in S:
        res["arms"][k] = summary(S[k], rf, ew)
        print(fmt(k, res["arms"][k]))
    print("=== books")
    for k, r in books.items():
        res["books"][k] = summary(r, rf, ew)
        print(fmt(k, res["books"][k]))
    mon = (1 + S).resample("ME").prod() - 1
    res["monthly_corr_svr_sleeve_vs_base"] = float(mon["svr_sleeve"].corr(mon["base"]))
    print(f"monthly corr svr_sleeve vs base {res['monthly_corr_svr_sleeve_vs_base']:.2f}")
    yr = (1 + S).groupby(S.index.year).prod() - 1
    res["yearly"] = {k: {int(y): float(v) for y, v in yr[k].items()} for k in yr}
    print("yearly %:\n", (yr * 100).round(1).to_string())

    a, b = res["arms"], res["books"]
    g = a[f"blend_w{W_GATED:.2f}"]
    checks = {
        "G1": g["sharpe"] >= a["base"]["sharpe"] + 0.10 and all(g["halves"][i] >= a["base"]["halves"][i] for i in range(2)),
        "G2": all(a[f"blend_w{w:.2f}"]["sharpe"] >= a["base"]["sharpe"] for w in W_NEIGHBOURS),
        "G3": g["mdd"] >= a["base"]["mdd"] - 0.03,
        "S1": all(b["book4"]["halves"][i] >= b["book3"]["halves"][i] + 0.05 for i in range(2)),
        "S2": b["book4"]["mdd"] >= b["book3"]["mdd"] - 0.03,
    }
    passing = [arm for arm, keys in (("blend", ("G1", "G2", "G3")), ("sleeve", ("S1", "S2")))
               if all(checks[k] for k in keys)]
    res["verdict"] = {"checks": checks, "passing": passing, "decision": "CONTINUE" if passing else "STOP"}
    print(f"checks {checks} -> {res['verdict']['decision']} {passing}")
    with open(res_path, "w") as fh:
        json.dump(res, fh, indent=2)
    S.join(pd.DataFrame(books)).to_pickle(os.path.join(OUT, "returns.pkl"))
    from research.lab.trials import append_trials
    append_trials(os.path.join(RESULTS, "trials_us.jsonl"),
                  [{"name": f"svr_combo:{k}", **{x: v for x, v in d.items() if x != "halves"}}
                   for grp in ("arms", "books") for k, d in res[grp].items()],
                  {"study": "svr_combo", "role": "preregistered consistency check"})


if __name__ == "__main__":
    main()