quant-trading/research/lab/managed_futures_book.py

"""
The conservative US book: does adding a managed-futures (futures trend)
sleeve to the live three-sleeve book give a steadier book with 20%+ a year
and a drawdown no worse than -30%, without lowering the Sharpe in either half
of 2006-2026?

    PYTHONPATH=. uv run --with numba python -m research.lab.stream_portfolio   # once: streams.pkl
    PYTHONPATH=. uv run python -m research.lab.futures_trend                   # once: returns.pkl
    PYTHONPATH=. uv run python -m research.lab.managed_futures_book [--no-proxy]

Streams (daily, costs inside; research/lab/stream_portfolio.py streams.pkl):
  momentum   overnight_recovery_top10, PIT S&P 500, lab engine
  lowrisk    lowrisk_top20 (ivol + low beta), lab engine
  trend_v7   trend_rider_v7_vt24 on synthetic 3x ETFs, "lag" fills (the live sleeve)
  trend_v7_vt36  trend_rider_v7_vt36, same engine (built here)
  trend      trend_rider_v10, same engine (reported only)
  mf         managed futures, the sleeve live trading can hold in a cash account:
             DBMF (iMGP DBi Managed Futures ETF, replicates the SG CTA index with
             futures; adjusted closes) from its first full day 2019-05-09; before
             that the 24-market futures TSMOM of research/lab/futures_trend.py
             (tsmom24) less DBMF's 0.85%/yr fee. The proxy is not DBMF: their
             monthly correlation over 2019-26 is 0.64.

Book: sleeves held as separate accounts in one book, as
strategies/sleeve_portfolio.py does live: shares drift with each sleeve's
return and are reset to the base shares at a month start when any share is
more than 25% (relative) off its base; 10 bp on the capital moved.

All of 2006-2026 was inspected in earlier studies (TrendRider was tuned on
2010-2026; the futures sleeve's blend was explored in futures_trend.py), so
this is a consistency gate, not out-of-sample evidence. The forward test is
the evidence. The rule is written to prereg.json before the run, and the
script refuses to overwrite results.json.

Result (2026-10-08): REJECT. Every candidate passes G2 and fails G1 and G3.
Excess Sharpe, 2006-10..2026-09, halves 2006-16 / 2017-26:

  | book                         | CAGR  | Sharpe | halves      | MaxDD | worst yr |
  |------------------------------|-------|--------|-------------|-------|----------|
  | three_sleeve_v7 (baseline)   | 18.9% | 0.98   | 0.697/1.332 | -37%  | -22.3%   |
  | four_sleeve_v7               | 16.3% | 1.01   | 0.763/1.301 | -27%  | -11.3%   |
  | four_sleeve_v7_vt36          | 17.2% | 1.00   | 0.723/1.325 | -29%  | -12.0%   |
  | three_sleeve_no_lowrisk      | 19.1% | 1.01   | 0.721/1.329 | -28%  |  -8.9%   |
  | four_sleeve_v10 (reported)   | 18.0% | 1.06   | 0.759/1.421 | -28%  | -10.4%   |
  | mf stream (proxy -> DBMF)    |  6.2% | 0.37   | 0.29/0.50   | -35%  | -25.4%   |

  - G1 misses in 2017-26 by 0.003-0.03 Sharpe: a tie, not an improvement.
  - G3: CAGR 16-19% is short of 20%.
  - The futures sleeve cuts MaxDD by about a quarter (-37% -> -27%) and
    halves the worst year (2008: -22% -> -9..-12%). It costs 0-2.6 pp of
    CAGR and leaves the Sharpe where it was (monthly corr with the other
    sleeves 0.01-0.24).
  - DBMF era, 2019-05..2026-09 (reported only): baseline 25.7% / 1.27 /
    -18%; four_sleeve_v7 21.9% / 1.28 / -15%; three_sleeve_no_lowrisk
    27.4% / 1.32 / -18%.
Interpretation: in a 1x cash account these streams give about 16-19% a year
at Sharpe ~1.0 over 2006-26. The "Sharpe 1.2, 20-30%" range came from
equal5 (with BTC) and the 2017-26 half, not from this book.
Decision: register nothing by this rule.
"""

from __future__ import annotations

import argparse
import json
import os
import pickle

import numpy as np
import pandas as pd

from research.lab.factor_scan import RESULTS
from research.lab.levered_book import stats, tbill

OUT = os.path.join(RESULTS, "managed_futures_book")
STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl")
FUTURES = os.path.join(RESULTS, "futures_trend", "returns.pkl")
SPAN = ("2006-10-02", "2026-09-24")
HALVES = [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")]
DBMF_START = "2019-05-09"
DBMF_FEE = 0.0085
BAND, COST = 0.25, 0.001

CANDIDATES = {
    "four_sleeve_v7": ["momentum", "lowrisk", "trend_v7", "mf"],
    "four_sleeve_v7_vt36": ["momentum", "lowrisk", "trend_v7_vt36", "mf"],
    "three_sleeve_no_lowrisk": ["momentum", "trend_v7", "mf"],
}
REPORTED = {
    "three_sleeve_v7": ["momentum", "lowrisk", "trend_v7"],
    "four_sleeve_v10": ["momentum", "lowrisk", "trend", "mf"],
    "four_sleeve_v7_tsmom_only": ["momentum", "lowrisk", "trend_v7", "tsmom24"],
    "four_sleeve_v7_etf_xtrend": ["momentum", "lowrisk", "trend_v7", "xtrend"],
}

PREREG = {
    "question": "Is momentum + lowrisk + TrendRider V7 + managed futures (equal shares) a steadier book "
                "with CAGR >= 20% and MaxDD >= -30%, no worse than the live three-sleeve book in either half?",
    "span": SPAN, "halves": HALVES,
    "candidates": CANDIDATES,
    "baseline": "three_sleeve_v7 (live us_three_sleeve: 1/3 momentum, lowrisk, trend_rider_v7_vt24), same blend",
    "mf_stream": f"DBMF from {DBMF_START}; before it tsmom24 (futures_trend.py) less {DBMF_FEE:.2%}/yr",
    "blend": f"equal base shares, drift, reset at a month start when any share is > {BAND:.0%} off; "
             f"{COST*1e4:.0f} bp on capital moved",
    "sharpe": "excess over the 3-month T-bill, daily, annualized",
    "gates_per_candidate": [
        "G1 Sharpe >= three_sleeve_v7 Sharpe in each half",
        "G2 full-span MaxDD >= -30%",
        "G3 full-span CAGR >= 20%",
    ],
    "selection": "among candidates passing G1-G3, the highest min-half Sharpe -> ACCEPT; else among those "
                 "passing G1-G2, the highest CAGR -> PARTIAL (meets the risk goal, not the return goal); "
                 "else REJECT",
    "decision": {"ACCEPT": "register the selected book and forward-test it as the conservative book",
                 "PARTIAL": "register it for forward testing, labeled as below the 20% return goal",
                 "REJECT": "register nothing; report which gate failed"},
    "reported_not_decisive": list(REPORTED) + ["DBMF-era (2019-05..) stats", "yearly returns",
                                               "monthly correlations"],
    "boundary": "all periods previously inspected; synthetic 3x ETFs before 2010; the mf proxy before "
                "2019-05 is a different strategy from DBMF",
}


def band_blend(R: pd.DataFrame, rf: pd.Series) -> tuple[pd.Series, pd.DataFrame]:
    """Book return of equal-share sleeves with drift and a banded month-start reset.

    A sleeve with no return yet (before its first observation) holds T-bills."""
    X = R.copy()
    for c in X:
        X[c] = X[c].fillna(rf.reindex(X.index).fillna(0.0) / 252)
    n = X.shape[1]
    base = np.full(n, 1.0 / n)
    s = base.copy()
    months = X.index.to_period("M")
    out, shares = np.empty(len(X)), np.empty((len(X), n))
    vals = X.to_numpy()
    for k in range(len(X)):
        cost = 0.0
        if k > 0 and months[k] != months[k - 1] and np.max(np.abs(s / base - 1)) > BAND:
            cost = np.abs(s - base).sum() * COST
            s = base.copy()
        shares[k] = s
        g = s * (1 + vals[k])
        out[k] = g.sum() - 1 - cost
        s = g / g.sum()
    return pd.Series(out, index=X.index), pd.DataFrame(shares, index=X.index, columns=X.columns)


def summary(r: pd.Series, rf: pd.Series) -> dict:
    r = r.loc[SPAN[0]:SPAN[1]]
    d = {k: float(v) for k, v in stats(r, rf).items()}
    d["halves"] = [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES]
    return d


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--no-proxy", action="store_true")
    if not ap.parse_args().no_proxy:
        from research.lab.fetch_us import _proxy_env
        _proxy_env()
    os.makedirs(OUT, exist_ok=True)
    res_path = os.path.join(OUT, "results.json")
    if os.path.exists(res_path):
        raise SystemExit(f"{res_path} exists; this study does not overwrite results")
    with open(os.path.join(OUT, "prereg.json"), "w") as fh:
        json.dump(PREREG, fh, indent=2)

    R = pd.read_pickle(STREAMS)
    days = R.index
    rf = tbill(days)
    from research.lab.fetch_long_history import fetch_yahoo
    from research.lab.letf_history import run as run_letf
    from trader import STRATEGY_REGISTRY
    with open(os.path.join("data", "research", "letf_history.pkl"), "rb") as fh:
        syn = pickle.load(fh)["close"].loc["1989-01-01":]
    R["trend_v7_vt36"] = run_letf(STRATEGY_REGISTRY["trend_rider_v7_vt36"](), syn, "lag").pct_change().reindex(days)
    fut = pd.read_pickle(FUTURES)["tsmom24"].reindex(days)
    dbmf_px, _ = fetch_yahoo(["DBMF"], start="2019-01-01")
    dbmf = dbmf_px["DBMF"].pct_change(fill_method=None).reindex(days).loc[DBMF_START:]
    R["tsmom24"] = fut
    R["mf"] = pd.concat([fut.loc[:DBMF_START].iloc[:-1] - DBMF_FEE / 252, dbmf]).reindex(days)

    B, avg_shares = {}, {}
    for name, cols in {**CANDIDATES, **REPORTED}.items():
        B[name], sh = band_blend(R.loc[SPAN[0]:SPAN[1], cols], rf)
        avg_shares[name] = sh.mean().round(3).to_dict()

    res = {"streams": {}, "books": {}}
    fmt = lambda k, d: (f"{k:28s} cagr {d['cagr']*100:5.1f}% xsh {d['sharpe']:5.2f} vol {d['vol']*100:4.1f}% "
                        f"mdd {d['mdd']*100:6.1f}% worst yr {d['worst_yr']*100:6.1f}% halves "
                        f"{d['halves'][0]:5.2f} {d['halves'][1]:5.2f}")
    print("=== streams")
    for c in ["momentum", "lowrisk", "trend_v7", "trend_v7_vt36", "trend", "mf", "tsmom24", "xtrend"]:
        res["streams"][c] = summary(R[c].dropna(), rf)
        print(fmt(c, res["streams"][c]))
    print("=== books")
    for k, r in B.items():
        res["books"][k] = summary(r, rf)
        print(fmt(k, res["books"][k]))

    # DBMF era: the real instrument, reported only
    res["dbmf_era"] = {}
    print(f"=== DBMF era {DBMF_START}..{SPAN[1]} (reported only)")
    for k in ["three_sleeve_v7", *CANDIDATES]:
        r = B[k].loc[DBMF_START:]
        res["dbmf_era"][k] = {x: float(v) for x, v in stats(r, rf).items()}
        print(f"  {k:28s} cagr {res['dbmf_era'][k]['cagr']*100:5.1f}% xsh {res['dbmf_era'][k]['sharpe']:5.2f} "
              f"mdd {res['dbmf_era'][k]['mdd']*100:6.1f}%")
    m = (1 + R.loc[SPAN[0]:SPAN[1], ["momentum", "lowrisk", "trend_v7", "mf"]].fillna(0.0)).resample("ME").prod() - 1
    print("monthly corr:\n", m.corr().round(2).to_string())
    res["monthly_corr"] = m.corr().round(3).to_dict()
    yr = pd.DataFrame({k: (1 + B[k]).groupby(B[k].index.year).prod() - 1
                       for k in ["three_sleeve_v7", *CANDIDATES]})
    print("yearly %:\n", (yr * 100).round(1).to_string())
    res["yearly"] = {k: {int(y): float(v) for y, v in yr[k].items()} for k in yr}
    res["avg_shares"] = avg_shares

    base = res["books"]["three_sleeve_v7"]["halves"]
    gates = {}
    for k in CANDIDATES:
        d = res["books"][k]
        gates[k] = {"G1": all(d["halves"][i] >= base[i] for i in range(2)),
                    "G2": d["mdd"] >= -0.30, "G3": d["cagr"] >= 0.20}
    full = [k for k, g in gates.items() if all(g.values())]
    risk = [k for k, g in gates.items() if g["G1"] and g["G2"]]
    if full:
        sel, decision = max(full, key=lambda k: min(res["books"][k]["halves"])), "ACCEPT"
    elif risk:
        sel, decision = max(risk, key=lambda k: res["books"][k]["cagr"]), "PARTIAL"
    else:
        sel, decision = None, "REJECT"
    res["verdict"] = {"gates": gates, "selected": sel, "decision": decision}
    print(f"gates {gates}\n-> {decision} {sel}")
    with open(res_path, "w") as fh:
        json.dump(res, fh, indent=2)
    pd.DataFrame(B).to_pickle(os.path.join(OUT, "books.pkl"))
    from research.lab.trials import append_trials
    append_trials(os.path.join(RESULTS, "trials_us.jsonl"),
                  [{"name": f"managed_futures_book:{k}", **v} for k, v in res["books"].items()],
                  {"study": "managed_futures_book", "role": "preregistered gate; all periods previously inspected"})


if __name__ == "__main__":
    main()