quant-trading/research/lab/overnight_book_mf.py

"""
At the leverage that takes the overnight books to 50% a year, does a
managed-futures (CTA) sleeve cut the drawdown enough to keep?

    PYTHONPATH=. uv run python -m research.lab.overnight_only        # once: daily.pkl
    PYTHONPATH=. uv run python -m research.lab.futures_trend         # once: returns.pkl
    PYTHONPATH=. uv run python -m research.lab.overnight_book_mf [--no-proxy]

Streams: overnight (overnight-only momentum, 1 bp a side), lowrisk, trend_v7
as in research/lab/overnight_book.py; mf as in
research/lab/managed_futures_book.py (DBMF from 2019-05-09, before it the
24-market futures TSMOM less 0.85%/yr; the proxy is not DBMF, monthly corr 0.64).

Books: equal base shares, banded month-start reset (managed_futures_book.band_blend).
Each book is levered to the L at which its 2006-10..2026-09 CAGR is 50%:
r_L = L x r - (L - 1) x (T-bill + 1%) / 252 (overnight_book.levered).

Pairs (candidate vs its base):
  four_sleeve_overnight_mf        overnight, lowrisk, trend_v7, mf  vs three_sleeve_overnight
  three_sleeve_overnight_trend_mf overnight, trend_v7, mf           vs two_sleeve_overnight_trend

All of 2006-2026 was inspected before (both the overnight books and the mf
sleeve). This is a consistency gate, not out-of-sample evidence. The rule is
written to prereg.json before the run; the script refuses to overwrite
results.json.

Result (2026-10-09): KEEP. Both candidates fail M1 and M2.

  book                             1x Sharpe  halves       L for 50%  MaxDD at L  worst yr at L
  three_sleeve_overnight           1.38       1.16 / 1.64  2.35       -36.7%      -18.6%
  four_sleeve_overnight_mf         1.36       1.15 / 1.58  2.93       -37.6%      -15.4%
  two_sleeve_overnight_trend       1.39       1.12 / 1.69  1.73       -34.8%      -24.3%
  three_sleeve_overnight_trend_mf  1.36       1.10 / 1.63  2.41       -36.4%      -20.8%

  - At 1x the mf sleeve trims MaxDD by 2-5 pp. It also costs 4-8 pp of
    CAGR, so the book needs 0.6-0.7x more leverage for 50%, and the levered
    MaxDD is unchanged.
  - The overnight books no longer have the drawdown CTA insures against:
    2008 at L is -2% / +18% without mf.
  - Their deepest drawdowns are 2008-06..11 (three-sleeve), 2014-07..2015-11
    (two-sleeve, a slow equity-momentum and trend slump) and 2020-02..03
    (with mf). Futures trend does not help in the first two.
"""

from __future__ import annotations

import argparse
import json
import os

import numpy as np
import pandas as pd

from research.lab.factor_scan import RESULTS
from research.lab.levered_book import stats, tbill
from research.lab.managed_futures_book import DBMF_FEE, DBMF_START, FUTURES, band_blend
from research.lab.overnight_book import HALVES, OVERNIGHT, SPAN, STREAMS, leverage_for, levered

OUT = os.path.join(RESULTS, "overnight_book_mf")
PAIRS = {
    "four_sleeve_overnight_mf": (["overnight", "lowrisk", "trend_v7", "mf"], "three_sleeve_overnight"),
    "three_sleeve_overnight_trend_mf": (["overnight", "trend_v7", "mf"], "two_sleeve_overnight_trend"),
}
BASES = {"three_sleeve_overnight": ["overnight", "lowrisk", "trend_v7"],
         "two_sleeve_overnight_trend": ["overnight", "trend_v7"]}

PREREG = {
    "question": "At the leverage giving 50%/yr, does adding a managed-futures sleeve to the overnight books "
                "cut MaxDD by >= 5 pp without lowering the 1x Sharpe?",
    "span": SPAN, "halves": HALVES, "pairs": {k: {"candidate": v[0], "base": v[1]} for k, v in PAIRS.items()},
    "bases": BASES,
    "mf_stream": f"DBMF from {DBMF_START}; before it tsmom24 less {DBMF_FEE:.2%}/yr",
    "leverage": "per book, L with full-span CAGR = 50%, borrowing at T-bill + 1%",
    "gates_per_candidate": ["M1 MaxDD at its L >= base MaxDD at the base's L + 5 pp",
                            "M2 1x excess Sharpe >= base 1x Sharpe - 0.05 in each half"],
    "decision": {"ADD_MF": "a candidate passes -> the book to forward-test includes a DBMF sleeve "
                           "(the one with the shallower levered MaxDD if both pass)",
                 "KEEP": "none passes -> no CTA sleeve; the overnight books stay as they are"},
    "reported": "worst year and vol at L, DBMF era (2019-05..) at L, yearly returns at L",
    "data_status": "all periods previously inspected; synthetic 3x ETFs before 2010; mf proxy before 2019-05",
}


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--no-proxy", action="store_true")
    if not ap.parse_args().no_proxy:
        from research.lab.fetch_us import _proxy_env
        _proxy_env()
    os.makedirs(OUT, exist_ok=True)
    res_path = os.path.join(OUT, "results.json")
    if os.path.exists(res_path):
        raise SystemExit(f"{res_path} exists; this study does not overwrite results")
    with open(os.path.join(OUT, "prereg.json"), "w") as fh:
        json.dump(PREREG, fh, indent=2)

    R = pd.read_pickle(STREAMS)
    days = R.index
    R["overnight"] = pd.read_pickle(OVERNIGHT)["sleeve_score"].reindex(days)
    from research.lab.fetch_long_history import fetch_yahoo
    fut = pd.read_pickle(FUTURES)["tsmom24"].reindex(days)
    dbmf_px, _ = fetch_yahoo(["DBMF"], start="2019-01-01")
    dbmf = dbmf_px["DBMF"].pct_change(fill_method=None).reindex(days).loc[DBMF_START:]
    R["mf"] = pd.concat([fut.loc[:DBMF_START].iloc[:-1] - DBMF_FEE / 252, dbmf]).reindex(days)
    R = R.loc[SPAN[0]:SPAN[1]]
    rf = tbill(R.index)

    books = {**BASES, **{k: v[0] for k, v in PAIRS.items()}}
    res = {"books": {}}
    B = {}
    for name, cols in books.items():
        r, _ = band_blend(R[cols], rf)
        B[name] = r
        one = stats(r, rf)
        L = leverage_for(r, rf)
        rl = levered(r, rf, L)
        s = stats(rl, rf)
        era = stats(rl.loc[DBMF_START:], rf)
        d = {"x1": {"cagr": float(one["cagr"]), "sharpe": float(one["sharpe"]), "mdd": float(one["mdd"]),
                    "halves": [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES]},
             "L": float(L), "at_L": {"mdd": float(s["mdd"]), "worst_yr": float(s["worst_yr"]), "vol": float(s["vol"]),
                                     "worst_3y": float(s["worst_3y"])},
             "dbmf_era_at_L": {"cagr": float(era["cagr"]), "sharpe": float(era["sharpe"]), "mdd": float(era["mdd"])},
             "yearly_at_L": {int(y): float(v) for y, v in ((1 + rl).groupby(rl.index.year).prod() - 1).items()}}
        res["books"][name] = d
        print(f"{name:34s} 1x cagr {d['x1']['cagr']*100:5.1f}% xsh {d['x1']['sharpe']:.2f} halves "
              f"{d['x1']['halves'][0]:.2f} {d['x1']['halves'][1]:.2f} mdd {d['x1']['mdd']*100:6.1f}% | L {L:.2f}: "
              f"mdd {s['mdd']*100:6.1f}% worst yr {s['worst_yr']*100:6.1f}% worst 3y/yr {s['worst_3y']*100:5.1f}% "
              f"vol {s['vol']*100:4.1f}% | DBMF era at L cagr {era['cagr']*100:5.1f}% xsh {era['sharpe']:.2f} "
              f"mdd {era['mdd']*100:6.1f}%", flush=True)

    yr = pd.DataFrame({k: res["books"][k]["yearly_at_L"] for k in books}) * 100
    print("yearly % at L:\n", yr.round(1).to_string())

    gates = {}
    for k, (_, base) in PAIRS.items():
        c, b = res["books"][k], res["books"][base]
        gates[k] = {"M1": c["at_L"]["mdd"] >= b["at_L"]["mdd"] + 0.05,
                    "M2": all(c["x1"]["halves"][i] >= b["x1"]["halves"][i] - 0.05 for i in range(2))}
    passing = [k for k, g in gates.items() if all(g.values())]
    sel = max(passing, key=lambda k: res["books"][k]["at_L"]["mdd"]) if passing else None
    decision = "ADD_MF" if passing else "KEEP"
    res["verdict"] = {"gates": gates, "passing": passing, "selected": sel, "decision": decision}
    print(f"gates {gates}\n-> {decision} {sel}")
    with open(res_path, "w") as fh:
        json.dump(res, fh, indent=2)
    pd.DataFrame(B).to_pickle(os.path.join(OUT, "books.pkl"))
    from research.lab.trials import append_trials
    append_trials(os.path.join(RESULTS, "trials_us.jsonl"),
                  [{"name": f"overnight_book_mf:{k}", "L": v["L"], **v["x1"], **{f"L_{x}": y for x, y in v["at_L"].items()}}
                   for k, v in res["books"].items()],
                  {"study": "overnight_book_mf", "role": "preregistered consistency gate"})


if __name__ == "__main__":
    main()