quant-trading/research/lab/futures_trend.py

"""
Quick check before building a futures pipeline: do free Yahoo continuous
futures (=F) support a backtest, and does a diversified futures trend book
(Moskowitz, Ooi & Pedersen 2012) add a better stream than the 8-ETF
long-only trend of research/lab/stream_portfolio.py (Sharpe 0.39)?

    PYTHONPATH=. uv run python -m research.lab.futures_trend [--no-proxy]

Data: Yahoo front-month continuous contracts. They are not back-adjusted, so
each roll shows up as a jump between contracts. Data checks first:
  tracking  daily corr and annual drift of ES/NQ/ZN/GC vs SPY/QQQ/IEF/GLD
            minus the T-bill (futures earn the excess return)
  jumps     days with |return| > 6x the 63-day vol, per market and year
Strategy (primary): 24 markets (4 equity, 4 rates, 6 FX, 3 metals,
4 energy, 3 grains). Position = sign(252-day return) x 40% / (EWMA-60 vol)
/ N, through t-1. Book return = T-bill + sum(w r), less 3 bp per unit of
notional traded. Jump days are kept as Yahoo prints them, and the
roll-cleaned variant (jumps zeroed) is reported next to it.

The decision rule is in PREREG and is written before the run; the script
refuses to overwrite results.json.

Result (2026-10-08): DATA. C1 and C2 fail; C3 passes.
  - Tracking: ES 0.979 and NQ 0.978 pass. ZN vs IEF (0.914) and GC vs GLD
    (0.888, drift +2.2%/yr) miss the bar. The likely cause is settlement
    times (ZN 15:00, GC 13:30 ET vs the ETFs' 16:00 close), not bad prints.
  - Roll jumps are rare: 0.0-0.7 days a year per market.
  - tsmom24: 7.4% / Sharpe 0.44, vol 15%, MaxDD -34%, halves 0.34 / 0.56.
    Correlation is 0.22 with TrendRider and 0.05 with momentum. Returns:
    +24.5% in 2008, +30.9% in 2022, -24.8% in 2016.
  - Roll-cleaned: 0.48 (0.34 / 0.65). Cleaning does not lift 2006-16, so
    better data is unlikely to reach 0.5 in both halves.
  - By sector: equity 0.44, metals 0.45, energy 0.27, grains 0.15, rates 0.11,
    FX -0.01.
  - Exploration afterwards (not preregistered, stream_portfolio blend):
    - tsmom24 in place of the ETF xtrend: invvol5 1.07 -> 1.21, equal5 1.23 -> 1.30.
    - Added to the three sleeves: 0.96-1.02 -> 1.05-1.07, MaxDD -32/-39% -> -20/-28%.
    It is a diversifier (crisis years) with a low stand-alone Sharpe, not a core stream.
"""

from __future__ import annotations

import argparse
import json
import os
import pickle

import numpy as np
import pandas as pd

from research.lab.factor_scan import RESULTS
from research.lab.fetch_long_history import fetch_yahoo
from research.lab.levered_book import stats, tbill

OUT = os.path.join(RESULTS, "futures_trend")
STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl")
MARKETS = {
    "equity": ["ES=F", "NQ=F", "YM=F", "RTY=F"],
    "rates": ["ZT=F", "ZF=F", "ZN=F", "ZB=F"],
    "fx": ["6E=F", "6J=F", "6B=F", "6A=F", "6C=F", "6S=F"],
    "metals": ["GC=F", "SI=F", "HG=F"],
    "energy": ["CL=F", "NG=F", "HO=F", "RB=F"],
    "grains": ["ZC=F", "ZW=F", "ZS=F"],
}
ALL = [t for v in MARKETS.values() for t in v]
TRACK = {"ES=F": "SPY", "NQ=F": "QQQ", "ZN=F": "IEF", "GC=F": "GLD"}
HALVES = [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")]
SPAN = (HALVES[0][0], HALVES[1][1])
COST = 0.0003

PREREG = {
    "question": "Is free Yahoo =F data good enough, and does 24-market futures TSMOM beat the ETF trend stream?",
    "primary": "tsmom24 (as printed by Yahoo)",
    "data_ok_if": "daily corr >= 0.95 for ES/NQ/ZN/GC vs their ETF and |annual drift| <= 2%",
    "continue_if_all": [
        "C1 data_ok",
        "C2 tsmom24 excess Sharpe >= 0.50 in both halves (2006-16, 2017-26)",
        "C3 monthly corr of tsmom24 with TrendRider V10 <= 0.40",
    ],
    "decision": {"CONTINUE": "buy roll-adjusted data (e.g. Norgate / CSI) and build a futures trend sleeve",
                 "STOP": "drop TODO 3 as a strategy source",
                 "DATA": "if C1 fails the strategy numbers are not trusted: decide on buying data alone"},
    "halves": HALVES,
}


def jumps(r: pd.Series) -> pd.Series:
    vol = r.rolling(63, min_periods=20).std().shift(1)
    return r.abs() > 6 * vol


def tsmom(R: pd.DataFrame, rf: pd.Series, look: int = 252) -> tuple[pd.Series, pd.Series]:
    lvl = (1 + R.fillna(0.0)).cumprod().where(R.notna())
    sig = np.sign(lvl / lvl.shift(look) - 1)
    vol = R.ewm(span=60, min_periods=60).std() * np.sqrt(252)
    n = sig.notna().sum(axis=1).clip(lower=1)
    W = (sig * (0.40 / vol)).div(n, axis=0).shift(1).fillna(0.0)
    ret = rf.reindex(R.index) / 252 + (W * R.fillna(0.0)).sum(axis=1) - W.diff().abs().sum(axis=1) * COST
    return ret, W.abs().sum(axis=1)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--no-proxy", action="store_true")
    if not ap.parse_args().no_proxy:
        from research.lab.fetch_us import _proxy_env
        _proxy_env()
    os.makedirs(OUT, exist_ok=True)
    res_path = os.path.join(OUT, "results.json")
    if os.path.exists(res_path):
        raise SystemExit(f"{res_path} exists; this study does not overwrite results")
    with open(os.path.join(OUT, "prereg.json"), "w") as fh:
        json.dump(PREREG, fh, indent=2)

    close, _ = fetch_yahoo(ALL, start="2000-01-01")
    missing = [t for t in ALL if t not in close or close[t].dropna().empty]
    print("missing:", missing)
    with open(os.path.join("data", "research", "long_history.pkl"), "rb") as fh:
        etf = pickle.load(fh)["close"]
    days = etf["SPY"].dropna().loc["2000-01-01":].index
    px = close.reindex(columns=[t for t in ALL if t not in missing]).reindex(days)
    R = px.pct_change(fill_method=None)
    rf = tbill(days)

    res = {"data": {}, "jumps_per_year": {}}
    print("\n=== tracking vs ETF (futures earn ETF return - T-bill)")
    for f, e in TRACK.items():
        if f in missing:
            continue
        a = R[f]
        b = etf[e].reindex(days).pct_change(fill_method=None) - rf / 252
        ok = a.notna() & b.notna()
        corr = float(a[ok].corr(b[ok]))
        drift = float((a[ok].mean() - b[ok].mean()) * 252)
        res["data"][f] = {"etf": e, "corr": corr, "annual_drift": drift, "start": str(a.first_valid_index().date())}
        print(f"  {f:6s} vs {e:4s} from {a.first_valid_index().date()} corr {corr:.3f} drift {drift*100:5.1f}%/yr")
    data_ok = all(v["corr"] >= 0.95 and abs(v["annual_drift"]) <= 0.02 for v in res["data"].values())

    print("\n=== jump days (|r| > 6x 63d vol) per year, and first date")
    J = R.apply(jumps)
    for t in px.columns:
        n = int(J[t].sum())
        yrs = max(R[t].notna().sum() / 252, 1)
        res["jumps_per_year"][t] = n / yrs
        print(f"  {t:6s} from {px[t].first_valid_index().date()} jumps {n:3d} ({n/yrs:4.1f}/yr)")

    book, gross = tsmom(R, rf)
    clean, _ = tsmom(R.mask(J, 0.0), rf)
    groups = {g: tsmom(R[[t for t in v if t in R]], rf)[0] for g, v in MARKETS.items()}
    S = pd.read_pickle(STREAMS)
    out = {"tsmom24": book, "tsmom24_roll_cleaned": clean, **{f"tsmom_{g}": r for g, r in groups.items()}}
    res["streams"] = {}
    print("\n=== strategies (excess Sharpe; halves 2006-16 / 2017-26)")
    m_tr = (1 + S["trend"].dropna()).resample("ME").prod() - 1
    m_mo = (1 + S["momentum"].dropna()).resample("ME").prod() - 1
    for k, r in {**out, "etf_xtrend": S["xtrend"].dropna(), "trend_v10": S["trend"].dropna()}.items():
        r = r.loc[SPAN[0]:SPAN[1]]
        s = stats(r, rf)
        h = [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES]
        m = (1 + r).resample("ME").prod() - 1
        c_tr, c_mo = float(m.corr(m_tr)), float(m.corr(m_mo))
        res["streams"][k] = {"cagr": float(s["cagr"]), "sharpe": float(s["sharpe"]), "vol": float(s["vol"]),
                             "mdd": float(s["mdd"]), "halves": h, "corr_trend": c_tr, "corr_momentum": c_mo}
        print(f"  {k:22s} cagr {s['cagr']*100:5.1f}% xsh {s['sharpe']:5.2f} vol {s['vol']*100:4.1f}% "
              f"mdd {s['mdd']*100:6.1f}% halves {h[0]:5.2f} {h[1]:5.2f} corr trend {c_tr:5.2f} mom {c_mo:5.2f}")
    print(f"  avg gross leverage tsmom24 {gross.loc[SPAN[0]:].mean():.2f}")
    yr = (1 + book.loc[SPAN[0]:SPAN[1]]).groupby(book.loc[SPAN[0]:SPAN[1]].index.year).prod() - 1
    print("tsmom24 yearly %:", (yr * 100).round(1).to_dict())
    res["yearly_tsmom24"] = {int(k): float(v) for k, v in yr.items()}

    p = res["streams"]["tsmom24"]
    checks = {"C1": data_ok, "C2": all(x >= 0.50 for x in p["halves"]), "C3": p["corr_trend"] <= 0.40}
    decision = "CONTINUE" if all(checks.values()) else ("DATA" if not data_ok else "STOP")
    res["verdict"] = {"checks": checks, "decision": decision}
    print(f"\nchecks {checks} -> {decision}")
    with open(res_path, "w") as fh:
        json.dump(res, fh, indent=2)
    pd.DataFrame(out).to_pickle(os.path.join(OUT, "returns.pkl"))
    from research.lab.trials import append_trials
    append_trials(os.path.join(RESULTS, "trials_us.jsonl"),
                  [{"name": f"futures_trend:{k}", **v} for k, v in res["streams"].items()],
                  {"study": "futures_trend", "role": "preregistered quick check; both halves previously inspected"})


if __name__ == "__main__":
    main()