""" Market-level backtests behind the web UI: ``bridge.py backtest`` (on demand) and ``precompute_backtest.py`` (the strategy page) share this one data path. * Universe: point-in-time index members in both markets (S&P 500 history from Wikipedia, CSI 300 history in data/csi300_history.json); prices outside a stock's membership are masked, so strategies cannot pick it then. * Book: kept like the live trader keeps it — its signal timing (``main.strategy_positions`` live mode), drifting holdings, the same order size floor and resize band (``main.simulate_live_book``). * Bars: panels a BarStrategy needs (open/high/low/volume) are loaded too. """ from __future__ import annotations import sys import pandas as pd import data_manager import universe_history as uh from main import simulate_live_book, strategy_positions from strategies.base import BarStrategy from trader import ( ETF_STRATEGY_UNIVERSES, LIVE_EXCLUDED_STRATEGIES, RESIZE_BAND, STRATEGY_REGISTRY, convert_quote_currency, filter_tradable_tickers, price_request, strategy_extra_tickers, strategy_fields, ) from universe import UNIVERSES # Rebalances below this size are not charged the per-trade fee in backtests, # matching the live trader's minimum order size. MIN_TRADE_VALUE = 50.0 def _log(msg: str) -> None: print(msg, file=sys.stderr) def trade_events(diff: pd.DataFrame, threshold: float = 0.01) -> list[dict]: """Trade events from per-ticker traded weights (rows = days).""" trades = [] for date in diff.index: row = diff.loc[date] significant = row[row.abs() > threshold] if significant.empty: continue buys = significant[significant > 0] sells = significant[significant < 0] if len(buys) > 0 or len(sells) > 0: trades.append({ "date": date.strftime("%Y-%m-%d"), "buys": len(buys), "sells": len(sells), "buyTickers": list(buys.nlargest(3).index), "sellTickers": list(sells.nsmallest(3).index), }) return trades def benchmark_curve(prices: pd.Series, dates: pd.DatetimeIndex, capital: float) -> list[dict]: """Benchmark value on ``dates``, rebased to ``capital`` on the first date it has a price.""" series = prices.reindex(dates).ffill().dropna() if series.empty: return [] values = series / series.iloc[0] * capital return [{"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)} for d, v in values.items()] def _slice_years(panels: dict[str, pd.DataFrame], years: int | None) -> dict[str, pd.DataFrame]: if not years: return panels close = panels["close"] cutoff = close.index[-1] - pd.DateOffset(years=years) return {f: p[p.index >= cutoff] for f, p in panels.items()} def load_stock_panels(market: str, strategy_names: list[str], years: int | None) -> dict: """PIT-masked panels for the market's stock strategies.""" universe = UNIVERSES[market] tickers = universe["fetch"]() benchmark = universe["benchmark"] since = None if years: since = (pd.Timestamp.today() - pd.DateOffset(years=years + 1)).strftime("%Y-%m-%d") ever = uh.pit_tickers_ever(market, since=since) or [] extras = {t for n in strategy_names for t in strategy_extra_tickers(n)} request = sorted(set(tickers) | set(ever) | extras | {benchmark}) fields = sorted({"close", "open"} | {f for n in strategy_names for f in strategy_fields(n)}) _log(f"PIT universe: {len(request)} tickers ({len(ever)} ever index members); fields {fields}") panels = data_manager.update(market, request, fields=fields) panels = _slice_years(panels, years) member = None if ever: member = uh.mask_prices_pit(panels["close"], market).notna() | panels["close"].isna() # Only tickers that were index members: the shared panel also holds # benchmarks and the ETFs of mixed strategies (TQQQ, SOXL, ...), which # the membership mask passes through unmasked. members_ever = set(ever) stocks = [t for t in panels["close"].columns if t in members_ever and t not in extras and panels["close"][t].where(member[t]).notna().any()] else: stocks = filter_tradable_tickers(panels["close"], tickers) return {"panels": panels, "member": member, "stocks": stocks, "benchmark": benchmark, "benchmark_label": universe["benchmark_label"]} def _run(strategy, panels: dict[str, pd.DataFrame], cols: list[str], capital: float, tx_cost: float, fixed_fee: float, fee_base: float, fee_per_share: float, member: pd.DataFrame | None = None, min_trade_value: float = MIN_TRADE_VALUE): """Close-price strategies see prices masked to index membership (they cannot pick a non-member); a BarStrategy sees full histories plus the membership as ``bars["eligible"]``. The book is valued on unmasked closes.""" close, opens = panels["close"][cols], panels["open"][cols] if isinstance(strategy, BarStrategy): bars = {f: panels[f][cols] for f in strategy.fields} if member is not None: bars["eligible"] = member.reindex(columns=cols).fillna(True) held = strategy_positions(strategy, close, open_data=opens, bars=bars) else: m = member.reindex(columns=cols).fillna(True) if member is not None else None held = strategy_positions(strategy, close.where(m) if m is not None else close, open_data=opens.where(m) if m is not None else opens) equity, traded = simulate_live_book(held, close, capital, tx_cost, fixed_fee, fee_base, fee_per_share, min_trade_value, RESIZE_BAND) return equity, trade_events(traded) def run_market(market: str, strategy_names: list[str], years: int | None, capital: float, tx_cost: float, fixed_fee: float, fee_base: float = 0.0, fee_per_share: float = 0.0) -> dict | None: """Backtest every strategy once; returns raw equity series and metadata.""" stock = [n for n in strategy_names if n.removeprefix("sim_") not in ETF_STRATEGY_UNIVERSES] etf = [n for n in strategy_names if n.removeprefix("sim_") in ETF_STRATEGY_UNIVERSES] fees = dict(tx_cost=tx_cost, fixed_fee=fixed_fee, fee_base=fee_base, fee_per_share=fee_per_share) runs: list[dict] = [] out = {"market": market, "benchmark_label": "SPY", "benchmark_curve": [], "start_date": None, "end_date": None, "tx_cost": tx_cost, "fixed_fee": fixed_fee} if stock: data = load_stock_panels(market, stock, years) panels, stocks = data["panels"], data["stocks"] close = panels["close"] _log(f"Universe: {len(stocks)} stocks, {close.index[0].date()} to {close.index[-1].date()}") for name in stock: _log(f" Running: {name}") try: strategy = STRATEGY_REGISTRY[name]() cols = stocks + [t for t in strategy_extra_tickers(name) if t in close.columns and t not in stocks] # Strategies the live book cannot hold (e.g. equal weight over # ~500 names) are shown without the live order-size floor. floor = 0.0 if name in LIVE_EXCLUDED_STRATEGIES else MIN_TRADE_VALUE equity, trades = _run(strategy, panels, cols, capital, **fees, member=data["member"], min_trade_value=floor) runs.append({"name": name, "equity": equity, "trades": trades}) _log(f" OK: {name} (final: {equity.iloc[-1]:,.0f})") except Exception as e: # noqa: BLE001 — one broken strategy must not sink the page _log(f" FAILED: {name}: {e}") out.update(benchmark_label=data["benchmark_label"], benchmark_curve=benchmark_curve(close[data["benchmark"]], close.index, capital), start_date=close.index[0].strftime("%Y-%m-%d"), end_date=close.index[-1].strftime("%Y-%m-%d")) for name in etf: _log(f" Running ETF strategy: {name}") try: etf_tickers = ETF_STRATEGY_UNIVERSES[name.removeprefix("sim_")] close, opens = data_manager.update( "etfs", price_request(market, sorted(set(etf_tickers + ["SPY"]))), with_open=True) panels = _slice_years({"close": convert_quote_currency(close), "open": convert_quote_currency(opens)}, years) strategy = STRATEGY_REGISTRY[name]() equity, trades = _run(strategy, panels, filter_tradable_tickers(panels["close"], etf_tickers), capital, **fees) runs.append({"name": name, "equity": equity, "trades": trades}) _log(f" OK: {name} (final: {equity.iloc[-1]:,.0f})") if not stock and not out["benchmark_curve"] and "SPY" in panels["close"].columns: spy = panels["close"]["SPY"] out.update(benchmark_label="SPY (Benchmark)", benchmark_curve=benchmark_curve(spy, spy.index, capital), start_date=spy.index[0].strftime("%Y-%m-%d"), end_date=spy.index[-1].strftime("%Y-%m-%d")) except Exception as e: # noqa: BLE001 _log(f" FAILED: {name}: {e}") if not runs: return None out["strategy_runs"] = runs return out