quant-trading/backtest_runner.py

"""
Market-level backtests behind the web UI: ``bridge.py backtest`` (on demand)
and ``precompute_backtest.py`` (the strategy page) share this one data path.

* Universe: point-in-time index members in both markets (S&P 500 history from
  Wikipedia, CSI 300 history in data/csi300_history.json); prices outside a
  stock's membership are masked, so strategies cannot pick it then.
* Book: kept like the live trader keeps it — its signal timing
  (``main.strategy_positions`` live mode), drifting holdings, the same order
  size floor and resize band (``main.simulate_live_book``).
* Bars: panels a BarStrategy needs (open/high/low/volume) are loaded too.
"""

from __future__ import annotations

import sys

import pandas as pd

import data_manager
import universe_history as uh
from main import simulate_live_book, strategy_positions
from strategies.base import BarStrategy
from trader import (
    ETF_STRATEGY_UNIVERSES,
    LIVE_EXCLUDED_STRATEGIES,
    RESIZE_BAND,
    STRATEGY_REGISTRY,
    convert_quote_currency,
    filter_tradable_tickers,
    price_request,
    strategy_extra_tickers,
    strategy_fields,
)
from universe import UNIVERSES

# Rebalances below this size are not charged the per-trade fee in backtests,
# matching the live trader's minimum order size.
MIN_TRADE_VALUE = 50.0


def _log(msg: str) -> None:
    print(msg, file=sys.stderr)


def trade_events(diff: pd.DataFrame, threshold: float = 0.01) -> list[dict]:
    """Trade events from per-ticker traded weights (rows = days)."""
    trades = []
    for date in diff.index:
        row = diff.loc[date]
        significant = row[row.abs() > threshold]
        if significant.empty:
            continue
        buys = significant[significant > 0]
        sells = significant[significant < 0]
        if len(buys) > 0 or len(sells) > 0:
            trades.append({
                "date": date.strftime("%Y-%m-%d"),
                "buys": len(buys),
                "sells": len(sells),
                "buyTickers": list(buys.nlargest(3).index),
                "sellTickers": list(sells.nsmallest(3).index),
            })
    return trades


def benchmark_curve(prices: pd.Series, dates: pd.DatetimeIndex, capital: float) -> list[dict]:
    """Benchmark value on ``dates``, rebased to ``capital`` on the first date it has a price."""
    series = prices.reindex(dates).ffill().dropna()
    if series.empty:
        return []
    values = series / series.iloc[0] * capital
    return [{"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)} for d, v in values.items()]


def _slice_years(panels: dict[str, pd.DataFrame], years: int | None) -> dict[str, pd.DataFrame]:
    if not years:
        return panels
    close = panels["close"]
    cutoff = close.index[-1] - pd.DateOffset(years=years)
    return {f: p[p.index >= cutoff] for f, p in panels.items()}


def load_stock_panels(market: str, strategy_names: list[str], years: int | None) -> dict:
    """PIT-masked panels for the market's stock strategies."""
    universe = UNIVERSES[market]
    tickers = universe["fetch"]()
    benchmark = universe["benchmark"]
    since = None
    if years:
        since = (pd.Timestamp.today() - pd.DateOffset(years=years + 1)).strftime("%Y-%m-%d")
    ever = uh.pit_tickers_ever(market, since=since) or []
    extras = {t for n in strategy_names for t in strategy_extra_tickers(n)}
    request = sorted(set(tickers) | set(ever) | extras | {benchmark})
    fields = sorted({"close", "open"} | {f for n in strategy_names for f in strategy_fields(n)})
    _log(f"PIT universe: {len(request)} tickers ({len(ever)} ever index members); fields {fields}")
    panels = data_manager.update(market, request, fields=fields)
    panels = _slice_years(panels, years)
    member = None
    if ever:
        member = uh.mask_prices_pit(panels["close"], market).notna() | panels["close"].isna()
        # Only tickers that were index members: the shared panel also holds
        # benchmarks and the ETFs of mixed strategies (TQQQ, SOXL, ...), which
        # the membership mask passes through unmasked.
        members_ever = set(ever)
        stocks = [t for t in panels["close"].columns
                  if t in members_ever and t not in extras
                  and panels["close"][t].where(member[t]).notna().any()]
    else:
        stocks = filter_tradable_tickers(panels["close"], tickers)
    return {"panels": panels, "member": member, "stocks": stocks, "benchmark": benchmark,
            "benchmark_label": universe["benchmark_label"]}


def _run(strategy, panels: dict[str, pd.DataFrame], cols: list[str], capital: float,
         tx_cost: float, fixed_fee: float, fee_base: float, fee_per_share: float,
         member: pd.DataFrame | None = None, min_trade_value: float = MIN_TRADE_VALUE):
    """Close-price strategies see prices masked to index membership (they
    cannot pick a non-member); a BarStrategy sees full histories plus the
    membership as ``bars["eligible"]``. The book is valued on unmasked closes."""
    close, opens = panels["close"][cols], panels["open"][cols]
    if isinstance(strategy, BarStrategy):
        bars = {f: panels[f][cols] for f in strategy.fields}
        if member is not None:
            bars["eligible"] = member.reindex(columns=cols).fillna(True)
        held = strategy_positions(strategy, close, open_data=opens, bars=bars)
    else:
        m = member.reindex(columns=cols).fillna(True) if member is not None else None
        held = strategy_positions(strategy, close.where(m) if m is not None else close,
                                  open_data=opens.where(m) if m is not None else opens)
    equity, traded = simulate_live_book(held, close, capital, tx_cost, fixed_fee, fee_base,
                                        fee_per_share, min_trade_value, RESIZE_BAND)
    return equity, trade_events(traded)


def run_market(market: str, strategy_names: list[str], years: int | None, capital: float,
               tx_cost: float, fixed_fee: float, fee_base: float = 0.0,
               fee_per_share: float = 0.0) -> dict | None:
    """Backtest every strategy once; returns raw equity series and metadata."""
    stock = [n for n in strategy_names if n.removeprefix("sim_") not in ETF_STRATEGY_UNIVERSES]
    etf = [n for n in strategy_names if n.removeprefix("sim_") in ETF_STRATEGY_UNIVERSES]
    fees = dict(tx_cost=tx_cost, fixed_fee=fixed_fee, fee_base=fee_base, fee_per_share=fee_per_share)
    runs: list[dict] = []
    out = {"market": market, "benchmark_label": "SPY", "benchmark_curve": [],
           "start_date": None, "end_date": None, "tx_cost": tx_cost, "fixed_fee": fixed_fee}

    if stock:
        data = load_stock_panels(market, stock, years)
        panels, stocks = data["panels"], data["stocks"]
        close = panels["close"]
        _log(f"Universe: {len(stocks)} stocks, {close.index[0].date()} to {close.index[-1].date()}")
        for name in stock:
            _log(f"  Running: {name}")
            try:
                strategy = STRATEGY_REGISTRY[name]()
                cols = stocks + [t for t in strategy_extra_tickers(name)
                                 if t in close.columns and t not in stocks]
                # Strategies the live book cannot hold (e.g. equal weight over
                # ~500 names) are shown without the live order-size floor.
                floor = 0.0 if name in LIVE_EXCLUDED_STRATEGIES else MIN_TRADE_VALUE
                equity, trades = _run(strategy, panels, cols, capital, **fees,
                                      member=data["member"], min_trade_value=floor)
                runs.append({"name": name, "equity": equity, "trades": trades})
                _log(f"    OK: {name} (final: {equity.iloc[-1]:,.0f})")
            except Exception as e:  # noqa: BLE001 — one broken strategy must not sink the page
                _log(f"    FAILED: {name}: {e}")
        out.update(benchmark_label=data["benchmark_label"],
                   benchmark_curve=benchmark_curve(close[data["benchmark"]], close.index, capital),
                   start_date=close.index[0].strftime("%Y-%m-%d"),
                   end_date=close.index[-1].strftime("%Y-%m-%d"))

    for name in etf:
        _log(f"  Running ETF strategy: {name}")
        try:
            etf_tickers = ETF_STRATEGY_UNIVERSES[name.removeprefix("sim_")]
            close, opens = data_manager.update(
                "etfs", price_request(market, sorted(set(etf_tickers + ["SPY"]))), with_open=True)
            panels = _slice_years({"close": convert_quote_currency(close),
                                   "open": convert_quote_currency(opens)}, years)
            strategy = STRATEGY_REGISTRY[name]()
            equity, trades = _run(strategy, panels, filter_tradable_tickers(panels["close"], etf_tickers),
                                  capital, **fees)
            runs.append({"name": name, "equity": equity, "trades": trades})
            _log(f"    OK: {name} (final: {equity.iloc[-1]:,.0f})")
            if not stock and not out["benchmark_curve"] and "SPY" in panels["close"].columns:
                spy = panels["close"]["SPY"]
                out.update(benchmark_label="SPY (Benchmark)",
                           benchmark_curve=benchmark_curve(spy, spy.index, capital),
                           start_date=spy.index[0].strftime("%Y-%m-%d"),
                           end_date=spy.index[-1].strftime("%Y-%m-%d"))
        except Exception as e:  # noqa: BLE001
            _log(f"    FAILED: {name}: {e}")

    if not runs:
        return None
    out["strategy_runs"] = runs
    return out