quant/research/v7_breakthrough_fixed.py

"""Fixed re-run for Directions B and C based on review feedback.

Direction B fix: recalibrate V3 thresholds per-sector (scale by vol ratio).
Direction C fix: monkey-patch V3._desired_regime inside real V7, preserving
                 the full state machine (confirm_days, cooloff, stop_loss, dd_stop).
"""
from __future__ import annotations

import sys
sys.path.insert(0, ".")

import numpy as np
import pandas as pd

import data_manager
import metrics
from main import backtest
from strategies.trend_rider_v7 import TrendRiderV7

YEARS = 10
CAPITAL = 100_000
TX_COST = 0.001
FIXED_FEE = 2.0


def load_etf_data():
    all_etfs = sorted(set([
        "SPY", "TQQQ", "UPRO", "GLD", "DBC", "SHY",
        "SOXL", "SMH", "TECL", "XLK", "TNA", "IWM", "FAS", "XLF",
    ]))
    data = data_manager.update("etfs", all_etfs, with_open=False)
    if isinstance(data, tuple):
        data = data[0]
    cutoff = data.index[-1] - pd.DateOffset(years=YEARS)
    return data[data.index >= cutoff]


def run(label, strategy, panel):
    eq = backtest(strategy, panel, initial_capital=CAPITAL,
                  transaction_cost=TX_COST, fixed_fee=FIXED_FEE)
    m = metrics.raw_summary(eq)
    print(f"  {label:<55} Ann={m['annualizedReturn']*100:>5.1f}%  "
          f"Sharpe={m['sharpeRatio']:.2f}  MaxDD={m['maxDrawdown']*100:.1f}%  "
          f"Sortino={m['sortinoRatio']:.2f}  Calmar={m['calmarRatio']:.2f}")
    return label, eq, m


# =========================================================================
# DIRECTION B FIX: per-sector calibrated thresholds
# =========================================================================

def direction_b_fixed(etf_data):
    print("\n" + "=" * 100)
    print("  DIRECTION B FIXED: Sector V7 with recalibrated thresholds")
    print("=" * 100)

    results = []
    core = [t for t in ["SPY", "TQQQ", "UPRO", "GLD", "DBC", "SHY"] if t in etf_data.columns]

    # Baseline
    r = run("V7+VT36 baseline (SPY→TQQQ/UPRO)", TrendRiderV7(target_vol=0.36, min_lev=0.75), etf_data[core])
    results.append(r)
    eq_v7 = r[1]

    # Estimate vol ratios for threshold scaling
    rets = etf_data.pct_change(fill_method=None).dropna()
    spy_vol = rets["SPY"].std() * np.sqrt(252) if "SPY" in rets.columns else 0.18
    print(f"\n  SPY realized vol: {spy_vol:.1%}")

    sector_configs = [
        ("SMH", ("SOXL",), "Semiconductor"),
        ("XLK", ("TECL",), "Technology"),
        ("IWM", ("TNA",), "Russell 2000"),
        ("XLF", ("FAS",), "Financials"),
    ]

    sector_eqs = {}
    for signal, risk_on, name in sector_configs:
        if signal not in etf_data.columns or risk_on[0] not in etf_data.columns:
            print(f"  SKIP {name}: missing data")
            continue

        sig_vol = rets[signal].std() * np.sqrt(252) if signal in rets.columns else spy_vol
        vol_ratio = sig_vol / spy_vol
        print(f"  {signal} vol: {sig_vol:.1%}, ratio to SPY: {vol_ratio:.2f}")

        needed = [signal] + list(risk_on) + ["GLD", "DBC", "SHY"]
        panel = etf_data[[t for t in needed if t in etf_data.columns]]

        # Uncalibrated (original V3 thresholds)
        v7_raw = TrendRiderV7(
            signal=signal, risk_on=risk_on, risk_off=("GLD", "DBC"),
            target_vol=0.36, min_lev=0.75,
        )
        r = run(f"  {name} UNCALIBRATED", v7_raw, panel)
        results.append(r)

        # Calibrated: scale vol/dd/peak thresholds by vol ratio
        v7_cal = TrendRiderV7(
            signal=signal, risk_on=risk_on, risk_off=("GLD", "DBC"),
            target_vol=0.36, min_lev=0.75,
            # V3 thresholds scaled by sector vol ratio
            vol_enter=0.14 * vol_ratio,
            vol_exit=0.20 * vol_ratio,
            dd_stop=0.05 * vol_ratio,
            peak_enter=0.02 * vol_ratio,
            peak_exit=0.05 * vol_ratio,
        )
        r = run(f"  {name} CALIBRATED (×{vol_ratio:.1f})", v7_cal, panel)
        results.append(r)
        sector_eqs[name] = r[1]

    # Ensembles with calibrated sectors
    if sector_eqs:
        print()
        for name, sec_eq in sector_eqs.items():
            for v7_pct in (0.5, 0.7):
                idx = eq_v7.index.intersection(sec_eq.index)
                v7_a = eq_v7.reindex(idx).ffill()
                sec_a = sec_eq.reindex(idx).ffill()
                ens = (v7_a / v7_a.iloc[0]) * v7_pct + (sec_a / sec_a.iloc[0]) * (1 - v7_pct)
                ens = ens * CAPITAL
                m = metrics.raw_summary(ens)
                label = f"  {int(v7_pct*100)}% SPY-V7 + {int((1-v7_pct)*100)}% {name[:8]}-V7 (cal)"
                print(f"  {label:<55} Ann={m['annualizedReturn']*100:>5.1f}%  "
                      f"Sharpe={m['sharpeRatio']:.2f}  MaxDD={m['maxDrawdown']*100:.1f}%  "
                      f"Sortino={m['sortinoRatio']:.2f}  Calmar={m['calmarRatio']:.2f}")
                results.append((label, ens, m))

    return results


# =========================================================================
# DIRECTION C FIX: inject alt regime into REAL V3 state machine
# =========================================================================

def direction_c_fixed(etf_data):
    print("\n" + "=" * 100)
    print("  DIRECTION C FIXED: Alt regimes inside real V3 state machine")
    print("=" * 100)

    core = [t for t in ["SPY", "TQQQ", "UPRO", "GLD", "DBC", "SHY"] if t in etf_data.columns]
    results = []

    # Baseline
    r = run("V7+VT36 (V3 full regime, baseline)", TrendRiderV7(target_vol=0.36, min_lev=0.75), etf_data[core])
    results.append(r)

    # Alt regimes: monkey-patch V3._desired_regime, preserving full FSM
    def make_alt_v7(regime_fn, label):
        v7 = TrendRiderV7(target_vol=0.36, min_lev=0.75)
        v7.v3._desired_regime = regime_fn
        return v7

    # --- Simple MA variants ---
    for window in (100, 150, 200, 250):
        def regime_ma(closes, current, w=window):
            if len(closes) < w:
                return "risk_off"
            return "risk_on" if closes[-1] > np.mean(closes[-w:]) else "risk_off"
        r = run(f"Simple MA{window}", make_alt_v7(regime_ma, f"MA{window}"), etf_data[core])
        results.append(r)

    # --- Dual MA crossover ---
    for short, long in ((50, 200), (50, 150), (20, 100)):
        def regime_dual(closes, current, s=short, l=long):
            if len(closes) < l:
                return "risk_off"
            return "risk_on" if np.mean(closes[-s:]) > np.mean(closes[-l:]) else "risk_off"
        r = run(f"Dual MA {short}/{long}", make_alt_v7(regime_dual, ""), etf_data[core])
        results.append(r)

    # --- ROC variants ---
    for window in (42, 63, 126):
        def regime_roc(closes, current, w=window):
            if len(closes) < w + 1 or closes[-w-1] <= 0:
                return "risk_off"
            return "risk_on" if closes[-1] / closes[-w-1] > 1.0 else "risk_off"
        r = run(f"ROC {window}d", make_alt_v7(regime_roc, ""), etf_data[core])
        results.append(r)

    # --- MA + vol filter (simplified V3) ---
    for ma_w, vol_cap in ((150, 0.20), (150, 0.25), (200, 0.22)):
        def regime_mavol(closes, current, mw=ma_w, vc=vol_cap):
            if len(closes) < max(mw, 21):
                return "risk_off"
            above = closes[-1] > np.mean(closes[-mw:])
            if not above:
                return "risk_off"
            rets = np.diff(closes[-21:]) / np.maximum(closes[-21:-1], 1e-12)
            vol = float(np.std(rets, ddof=1) * np.sqrt(252))
            return "risk_on" if vol < vc else "risk_off"
        r = run(f"MA{ma_w} + Vol<{int(vol_cap*100)}%", make_alt_v7(regime_mavol, ""), etf_data[core])
        results.append(r)

    # --- Composite (MA + ROC + vol) ---
    for thresh in (2, 3):
        def regime_comp(closes, current, t=thresh):
            if len(closes) < 200:
                return "risk_off"
            score = 0
            if closes[-1] > np.mean(closes[-150:]):
                score += 1
            if closes[-64] > 0 and closes[-1] / closes[-64] > 1.0:
                score += 1
            rets = np.diff(closes[-21:]) / np.maximum(closes[-21:-1], 1e-12)
            if np.std(rets, ddof=1) * np.sqrt(252) < 0.22:
                score += 1
            return "risk_on" if score >= t else "risk_off"
        r = run(f"Composite {thresh}/3", make_alt_v7(regime_comp, ""), etf_data[core])
        results.append(r)

    # --- MA + slope (MA must be rising) ---
    for slope_w in (10, 20):
        def regime_slope(closes, current, sw=slope_w):
            if len(closes) < 150 + sw:
                return "risk_off"
            ma_now = np.mean(closes[-150:])
            ma_prev = np.mean(closes[-150-sw:-sw])
            return "risk_on" if (closes[-1] > ma_now and ma_now > ma_prev) else "risk_off"
        r = run(f"MA150 + Rising({slope_w}d)", make_alt_v7(regime_slope, ""), etf_data[core])
        results.append(r)

    # --- Adaptive MA (fast in low vol, slow in high vol) ---
    for pivot in (0.15, 0.18, 0.22):
        def regime_adapt(closes, current, p=pivot):
            if len(closes) < 200:
                return "risk_off"
            rets = np.diff(closes[-61:]) / np.maximum(closes[-61:-1], 1e-12)
            vol = np.std(rets, ddof=1) * np.sqrt(252)
            w = 200 if vol > p else 100
            return "risk_on" if closes[-1] > np.mean(closes[-w:]) else "risk_off"
        r = run(f"Adaptive MA (pivot={int(pivot*100)}%)", make_alt_v7(regime_adapt, ""), etf_data[core])
        results.append(r)

    # Sort and display
    results.sort(key=lambda x: x[2]["sharpeRatio"], reverse=True)
    print(f"\n--- Direction C FIXED Results (sorted by Sharpe) ---")
    for i, (label, _, m) in enumerate(results, 1):
        marker = " ★" if i <= 3 else ""
        print(f"  {i:<3} {label:<55} Ann={m['annualizedReturn']*100:>5.1f}%  "
              f"Sharpe={m['sharpeRatio']:.2f}  MaxDD={m['maxDrawdown']*100:.1f}%  "
              f"Calmar={m['calmarRatio']:.2f}{marker}")

    return results


def main():
    print("=" * 100)
    print("  V7 BREAKTHROUGH EVAL — FIXED RE-RUN (per review feedback)")
    print("=" * 100)

    etf_data = load_etf_data()
    print(f"Period: {etf_data.index[0].date()} → {etf_data.index[-1].date()}")
    print(f"ETFs: {sorted(etf_data.columns.tolist())}")

    results_b = direction_b_fixed(etf_data)
    results_c = direction_c_fixed(etf_data)

    # Cross-direction top 10
    all_r = [(f"[B] {l}", eq, m) for l, eq, m in results_b] + \
            [(f"[C] {l}", eq, m) for l, eq, m in results_c]
    all_r.sort(key=lambda x: x[2]["sharpeRatio"], reverse=True)
    print(f"\n{'=' * 100}")
    print("  FINAL: Top 10 by Sharpe")
    print(f"{'=' * 100}")
    for i, (label, _, m) in enumerate(all_r[:10], 1):
        print(f"  {i:<3} {label:<60} Ann={m['annualizedReturn']*100:>5.1f}%  "
              f"Sharpe={m['sharpeRatio']:.2f}  MaxDD={m['maxDrawdown']*100:.1f}%  "
              f"Calmar={m['calmarRatio']:.2f}")

    all_r.sort(key=lambda x: x[2]["annualizedReturn"], reverse=True)
    print(f"\n  FINAL: Top 10 by Ann. Return")
    print(f"  {'-' * 95}")
    for i, (label, _, m) in enumerate(all_r[:10], 1):
        print(f"  {i:<3} {label:<60} Ann={m['annualizedReturn']*100:>5.1f}%  "
              f"Sharpe={m['sharpeRatio']:.2f}  MaxDD={m['maxDrawdown']*100:.1f}%  "
              f"Calmar={m['calmarRatio']:.2f}")


if __name__ == "__main__":
    main()