""" Quick check before building a futures pipeline: do free Yahoo continuous futures (=F) support a backtest, and does a diversified futures trend book (Moskowitz, Ooi & Pedersen 2012) add a better stream than the 8-ETF long-only trend of research/lab/stream_portfolio.py (Sharpe 0.39)? PYTHONPATH=. uv run python -m research.lab.futures_trend [--no-proxy] Data: Yahoo front-month continuous contracts. They are not back-adjusted, so each roll shows up as a jump between contracts. Data checks first: tracking daily corr and annual drift of ES/NQ/ZN/GC vs SPY/QQQ/IEF/GLD minus the T-bill (futures earn the excess return) jumps days with |return| > 6x the 63-day vol, per market and year Strategy (primary): 24 markets (4 equity, 4 rates, 6 FX, 3 metals, 4 energy, 3 grains). Position = sign(252-day return) x 40% / (EWMA-60 vol) / N, through t-1. Book return = T-bill + sum(w r), less 3 bp per unit of notional traded. Jump days are kept as Yahoo prints them, and the roll-cleaned variant (jumps zeroed) is reported next to it. The decision rule is in PREREG and is written before the run; the script refuses to overwrite results.json. Result (2026-10-08): DATA. C1 and C2 fail; C3 passes. - Tracking: ES 0.979 and NQ 0.978 pass. ZN vs IEF (0.914) and GC vs GLD (0.888, drift +2.2%/yr) miss the bar. The likely cause is settlement times (ZN 15:00, GC 13:30 ET vs the ETFs' 16:00 close), not bad prints. - Roll jumps are rare: 0.0-0.7 days a year per market. - tsmom24: 7.4% / Sharpe 0.44, vol 15%, MaxDD -34%, halves 0.34 / 0.56. Correlation is 0.22 with TrendRider and 0.05 with momentum. Returns: +24.5% in 2008, +30.9% in 2022, -24.8% in 2016. - Roll-cleaned: 0.48 (0.34 / 0.65). Cleaning does not lift 2006-16, so better data is unlikely to reach 0.5 in both halves. - By sector: equity 0.44, metals 0.45, energy 0.27, grains 0.15, rates 0.11, FX -0.01. - Exploration afterwards (not preregistered, stream_portfolio blend): - tsmom24 in place of the ETF xtrend: invvol5 1.07 -> 1.21, equal5 1.23 -> 1.30. - Added to the three sleeves: 0.96-1.02 -> 1.05-1.07, MaxDD -32/-39% -> -20/-28%. It is a diversifier (crisis years) with a low stand-alone Sharpe, not a core stream. """ from __future__ import annotations import argparse import json import os import pickle import numpy as np import pandas as pd from research.lab.factor_scan import RESULTS from research.lab.fetch_long_history import fetch_yahoo from research.lab.levered_book import stats, tbill OUT = os.path.join(RESULTS, "futures_trend") STREAMS = os.path.join(RESULTS, "stream_portfolio", "streams.pkl") MARKETS = { "equity": ["ES=F", "NQ=F", "YM=F", "RTY=F"], "rates": ["ZT=F", "ZF=F", "ZN=F", "ZB=F"], "fx": ["6E=F", "6J=F", "6B=F", "6A=F", "6C=F", "6S=F"], "metals": ["GC=F", "SI=F", "HG=F"], "energy": ["CL=F", "NG=F", "HO=F", "RB=F"], "grains": ["ZC=F", "ZW=F", "ZS=F"], } ALL = [t for v in MARKETS.values() for t in v] TRACK = {"ES=F": "SPY", "NQ=F": "QQQ", "ZN=F": "IEF", "GC=F": "GLD"} HALVES = [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")] SPAN = (HALVES[0][0], HALVES[1][1]) COST = 0.0003 PREREG = { "question": "Is free Yahoo =F data good enough, and does 24-market futures TSMOM beat the ETF trend stream?", "primary": "tsmom24 (as printed by Yahoo)", "data_ok_if": "daily corr >= 0.95 for ES/NQ/ZN/GC vs their ETF and |annual drift| <= 2%", "continue_if_all": [ "C1 data_ok", "C2 tsmom24 excess Sharpe >= 0.50 in both halves (2006-16, 2017-26)", "C3 monthly corr of tsmom24 with TrendRider V10 <= 0.40", ], "decision": {"CONTINUE": "buy roll-adjusted data (e.g. Norgate / CSI) and build a futures trend sleeve", "STOP": "drop TODO 3 as a strategy source", "DATA": "if C1 fails the strategy numbers are not trusted: decide on buying data alone"}, "halves": HALVES, } def jumps(r: pd.Series) -> pd.Series: vol = r.rolling(63, min_periods=20).std().shift(1) return r.abs() > 6 * vol def tsmom(R: pd.DataFrame, rf: pd.Series, look: int = 252) -> tuple[pd.Series, pd.Series]: lvl = (1 + R.fillna(0.0)).cumprod().where(R.notna()) sig = np.sign(lvl / lvl.shift(look) - 1) vol = R.ewm(span=60, min_periods=60).std() * np.sqrt(252) n = sig.notna().sum(axis=1).clip(lower=1) W = (sig * (0.40 / vol)).div(n, axis=0).shift(1).fillna(0.0) ret = rf.reindex(R.index) / 252 + (W * R.fillna(0.0)).sum(axis=1) - W.diff().abs().sum(axis=1) * COST return ret, W.abs().sum(axis=1) def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--no-proxy", action="store_true") if not ap.parse_args().no_proxy: from research.lab.fetch_us import _proxy_env _proxy_env() os.makedirs(OUT, exist_ok=True) res_path = os.path.join(OUT, "results.json") if os.path.exists(res_path): raise SystemExit(f"{res_path} exists; this study does not overwrite results") with open(os.path.join(OUT, "prereg.json"), "w") as fh: json.dump(PREREG, fh, indent=2) close, _ = fetch_yahoo(ALL, start="2000-01-01") missing = [t for t in ALL if t not in close or close[t].dropna().empty] print("missing:", missing) with open(os.path.join("data", "research", "long_history.pkl"), "rb") as fh: etf = pickle.load(fh)["close"] days = etf["SPY"].dropna().loc["2000-01-01":].index px = close.reindex(columns=[t for t in ALL if t not in missing]).reindex(days) R = px.pct_change(fill_method=None) rf = tbill(days) res = {"data": {}, "jumps_per_year": {}} print("\n=== tracking vs ETF (futures earn ETF return - T-bill)") for f, e in TRACK.items(): if f in missing: continue a = R[f] b = etf[e].reindex(days).pct_change(fill_method=None) - rf / 252 ok = a.notna() & b.notna() corr = float(a[ok].corr(b[ok])) drift = float((a[ok].mean() - b[ok].mean()) * 252) res["data"][f] = {"etf": e, "corr": corr, "annual_drift": drift, "start": str(a.first_valid_index().date())} print(f" {f:6s} vs {e:4s} from {a.first_valid_index().date()} corr {corr:.3f} drift {drift*100:5.1f}%/yr") data_ok = all(v["corr"] >= 0.95 and abs(v["annual_drift"]) <= 0.02 for v in res["data"].values()) print("\n=== jump days (|r| > 6x 63d vol) per year, and first date") J = R.apply(jumps) for t in px.columns: n = int(J[t].sum()) yrs = max(R[t].notna().sum() / 252, 1) res["jumps_per_year"][t] = n / yrs print(f" {t:6s} from {px[t].first_valid_index().date()} jumps {n:3d} ({n/yrs:4.1f}/yr)") book, gross = tsmom(R, rf) clean, _ = tsmom(R.mask(J, 0.0), rf) groups = {g: tsmom(R[[t for t in v if t in R]], rf)[0] for g, v in MARKETS.items()} S = pd.read_pickle(STREAMS) out = {"tsmom24": book, "tsmom24_roll_cleaned": clean, **{f"tsmom_{g}": r for g, r in groups.items()}} res["streams"] = {} print("\n=== strategies (excess Sharpe; halves 2006-16 / 2017-26)") m_tr = (1 + S["trend"].dropna()).resample("ME").prod() - 1 m_mo = (1 + S["momentum"].dropna()).resample("ME").prod() - 1 for k, r in {**out, "etf_xtrend": S["xtrend"].dropna(), "trend_v10": S["trend"].dropna()}.items(): r = r.loc[SPAN[0]:SPAN[1]] s = stats(r, rf) h = [float(stats(r.loc[a:b], rf)["sharpe"]) for a, b in HALVES] m = (1 + r).resample("ME").prod() - 1 c_tr, c_mo = float(m.corr(m_tr)), float(m.corr(m_mo)) res["streams"][k] = {"cagr": float(s["cagr"]), "sharpe": float(s["sharpe"]), "vol": float(s["vol"]), "mdd": float(s["mdd"]), "halves": h, "corr_trend": c_tr, "corr_momentum": c_mo} print(f" {k:22s} cagr {s['cagr']*100:5.1f}% xsh {s['sharpe']:5.2f} vol {s['vol']*100:4.1f}% " f"mdd {s['mdd']*100:6.1f}% halves {h[0]:5.2f} {h[1]:5.2f} corr trend {c_tr:5.2f} mom {c_mo:5.2f}") print(f" avg gross leverage tsmom24 {gross.loc[SPAN[0]:].mean():.2f}") yr = (1 + book.loc[SPAN[0]:SPAN[1]]).groupby(book.loc[SPAN[0]:SPAN[1]].index.year).prod() - 1 print("tsmom24 yearly %:", (yr * 100).round(1).to_dict()) res["yearly_tsmom24"] = {int(k): float(v) for k, v in yr.items()} p = res["streams"]["tsmom24"] checks = {"C1": data_ok, "C2": all(x >= 0.50 for x in p["halves"]), "C3": p["corr_trend"] <= 0.40} decision = "CONTINUE" if all(checks.values()) else ("DATA" if not data_ok else "STOP") res["verdict"] = {"checks": checks, "decision": decision} print(f"\nchecks {checks} -> {decision}") with open(res_path, "w") as fh: json.dump(res, fh, indent=2) pd.DataFrame(out).to_pickle(os.path.join(OUT, "returns.pkl")) from research.lab.trials import append_trials append_trials(os.path.join(RESULTS, "trials_us.jsonl"), [{"name": f"futures_trend:{k}", **v} for k, v in res["streams"].items()], {"study": "futures_trend", "role": "preregistered quick check; both halves previously inspected"}) if __name__ == "__main__": main()