quant-trading/research/lab/beat_qqq.py

"""
Can a US book beat QQQ with alpha that holds out of sample? Assembles the
sleeves that showed alpha against QQQ and checks them on 2017-2026 with
choices made on 2006-2016 only.

    PYTHONPATH=. uv run --with numba python -m research.lab.qqq_scan      # factor alphas
    PYTHONPATH=. uv run python -m research.lab.vol_trend                  # QQQ trend grid
    PYTHONPATH=. uv run --with numba python -m research.lab.beat_qqq

Sleeves (all long-only, 10 bp costs, next-open fills):
  momentum   overnight-return momentum + distance from 126d low, top 10, every
             10 days, buffer 2 (the live overnight_recovery_top10)
  quality    mean rank of the K fundamental factors with the highest design-
             period (2006-2016) alpha t-stat against QQQ in qqq_scan_factors.csv,
             top 20, monthly, buffer 2. K and the list are fixed before the
             holdout is looked at.
  qqq_trend  QQQ exposure = 1{QQQ > 200-day average} x min(1.5, 30% / EWMA vol),
             held as TQQQ + 2-year Treasuries (research/lab/vol_trend.py; the
             200-day rule and a vol target near QQQ's own are the textbook
             choices, not fitted)
Books are equal capital, reset monthly. Everything is reported against QQQ:
alpha (Newey-West t), CAGR per half, MaxDD, and the calendar years won.
"""

from __future__ import annotations

import json
import os
import pickle
import warnings

import numpy as np
import pandas as pd

from research.lab import fundamentals as fu
from research.lab.alpha import alpha_stats, bench_returns, fmt_alpha
from research.lab.factor_scan import RESULTS, load
from research.lab.rotation import factor_ranks
from research.lab.us_final import monthly_blend
from research.lab.us_sweep import _blend, run, sector_rank

warnings.filterwarnings("ignore")
LONG = ("2006-09-27", "2026-09-24")
DESIGN_END = "2016-12-31"


def pick_fundamentals(k: int = 5) -> list[tuple[str, bool]]:
    """(factor, sector-neutral) with the best design-period alpha t vs QQQ,
    one variant per factor."""
    d = pd.read_csv(os.path.join(RESULTS, "qqq_scan_factors.csv"))
    d = d[d["name"].str.startswith("f:")].sort_values("design_t", ascending=False)
    out, seen = [], set()
    for name in d["name"]:
        base = name[2:].replace(" (sector)", "")
        if base in seen:
            continue
        seen.add(base)
        out.append((base, name.endswith("(sector)")))
        if len(out) == k:
            break
    return out


def sleeves(p) -> tuple[pd.DataFrame, list]:
    pr = factor_ranks(p, "us")
    F = fu.load(p)
    picks = pick_fundamentals()
    ranks = [sector_rank(F[f], p) if sec else F[f].where(p.member).rank(axis=1, pct=True) for f, sec in picks]
    quality = _blend(*ranks)
    mom = _blend(pr["overnight_mom"], pr["rec_126"])
    R = {"momentum": run(p, mom.where(p.member), [LONG], top_n=10, buffer=2.0, every=10)["_rets"],
         "quality": run(p, quality.where(p.member), [LONG], top_n=20, buffer=2.0, every=21)["_rets"]}
    with open(os.path.join(RESULTS, "vol_trend.pkl"), "rb") as fh:
        R["qqq_trend"] = pickle.load(fh)["returns"]["QQQ ma200 vt30 cap1.5"].reindex(p.dates)
    return pd.DataFrame(R).loc[LONG[0]:LONG[1]].fillna(0.0), picks


def show(name: str, r: pd.Series, b: pd.Series) -> dict:
    a, d, h = alpha_stats(r, b), alpha_stats(r.loc[:DESIGN_END], b), alpha_stats(r.loc["2017":], b)
    e = (1 + r).cumprod()
    mdd = float((e / e.cummax() - 1).min())
    print(f"{fmt_alpha(name, a)} mdd {mdd*100:4.0f}% | 06-16 a {d['alpha']*100:5.1f}% t {d['t']:5.2f} "
          f"cagr {d['cagr']*100:5.1f} vs {d['bench_cagr']*100:5.1f} | 17-26 a {h['alpha']*100:5.1f}% "
          f"t {h['t']:5.2f} cagr {h['cagr']*100:5.1f} vs {h['bench_cagr']*100:5.1f}")
    return {"full": a, "design": d, "holdout": h, "mdd": mdd}


def main() -> None:
    p, _ = load("us")
    R, picks = sleeves(p)
    b = bench_returns(R.index)
    print("fundamentals picked on 2006-2016 alpha t:", picks)
    m = (1 + R).resample("ME").prod() - 1
    print("monthly correlation:\n", m.corr().round(2).to_string())
    out = {"picks": picks}
    for c in R.columns:
        out[c] = show(c, R[c], b)
    books = {"momentum + qqq_trend": {"momentum": .5, "qqq_trend": .5},
             "momentum + quality": {"momentum": .5, "quality": .5},
             "momentum + quality + qqq_trend": {"momentum": 1 / 3, "quality": 1 / 3, "qqq_trend": 1 / 3},
             "momentum .5 + quality .25 + qqq_trend .25": {"momentum": .5, "quality": .25, "qqq_trend": .25}}
    for name, w in books.items():
        r = monthly_blend(R, w)
        out[name] = show(name, r, b)
    r = monthly_blend(R, books["momentum + quality + qqq_trend"])
    yr = pd.DataFrame({"book": (1 + r).groupby(r.index.year).prod() - 1,
                       "QQQ": (1 + b).groupby(b.index.year).prod() - 1})
    print("\nyearly, momentum + quality + qqq_trend vs QQQ (%):")
    print((yr * 100).round(1).assign(beat=yr["book"] > yr["QQQ"]).to_string())
    print("\nodds of beating QQQ over a horizon (historical rolling windows | stationary block bootstrap, 63-day blocks)")
    cands = {"momentum": R["momentum"], "qqq_trend": R["qqq_trend"],
             "momentum + qqq_trend": monthly_blend(R, books["momentum + qqq_trend"]),
             "momentum + quality + qqq_trend": r}
    rng = np.random.default_rng(0)
    for name, x in cands.items():
        cells = []
        for years in (1, 3, 5):
            h = 252 * years
            lx, lb = np.log1p(x.to_numpy()), np.log1p(b.reindex(x.index).fillna(0.0).to_numpy())
            cx, cb = np.r_[0, np.cumsum(lx)], np.r_[0, np.cumsum(lb)]
            hist = float(((cx[h:] - cx[:-h]) > (cb[h:] - cb[:-h])).mean())
            wins = 0
            for _ in range(2000):
                idx = []
                while len(idx) < h:
                    s = rng.integers(0, len(lx))
                    idx.extend(range(s, min(s + rng.geometric(1 / 63), len(lx))))
                idx = idx[:h]
                wins += lx[idx].sum() > lb[idx].sum()
            cells.append(f"{years}y {hist*100:3.0f}% | {wins / 20:3.0f}%")
        print(f"  {name:32s} " + "   ".join(cells))

    with open(os.path.join(RESULTS, "beat_qqq.json"), "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    R.to_pickle(os.path.join(RESULTS, "beat_qqq_sleeves.pkl"))


if __name__ == "__main__":
    main()