Gahow Wang
cc1366b2dc
Strategies - BarStrategy protocol (close/open/high/low/volume bars; row d uses bars through d-1) and RobustMomentumStrategy: overnight_recovery_top10 (US), lowrisk_momentum_top15 and lowbeta_sharpe_mom_top20 (CN), top-N with rank hysteresis, 10-day rebalance, weights drift between rebalances. - trader: strategy_fields/strategy_bars/bar_session_weights; one signal path (_signals_for_sessions) replaces generate_target_weights. Live fixes - CSI 300 constituents come from China Securities Index's file; the Wikipedia table was 122 names off the real index. - S&P 500 change log moved to "Historical components of the S&P 500". - data_manager: tickers missing from a partial Yahoo batch are retried alone before being marked unavailable (CEG, GEV, SNDK ... were dropped). update() takes fields=[...] and stores high/low/volume panels. - Whole-lot orders go to the nearest lot within cash (CN books held ~62% cash); held names are resized only beyond RESIZE_BAND (10%), so books no longer pay a ticket fee to trade daily drift. Web backtests - backtest_runner.py replaces the duplicated bridge/precompute code: PIT membership in both markets (data/csi300_history.json), stock universe limited to index members (ETFs and benchmarks in the shared panel were pickable), and the book kept like the live trader (simulate_live_book: signal timing, drift, order floor, resize band, fees on orders placed). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
427 lines
19 KiB
Python
427 lines
19 KiB
Python
import argparse
|
|
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
import data_manager
|
|
import factor_attribution
|
|
import metrics
|
|
import universe_history as uh
|
|
from strategies.adaptive_momentum import AdaptiveMomentumStrategy
|
|
from strategies.base import BarStrategy
|
|
from strategies.buy_and_hold import BuyAndHoldStrategy
|
|
from strategies.dual_momentum import DualMomentumStrategy
|
|
from strategies.inverse_vol import InverseVolatilityStrategy
|
|
from strategies.mean_reversion import MeanReversionStrategy
|
|
from strategies.momentum import MomentumStrategy
|
|
from strategies.momentum_quality import MomentumQualityStrategy
|
|
from strategies.multi_factor import MultiFactorStrategy
|
|
from strategies.recovery_momentum import RecoveryMomentumStrategy
|
|
from strategies.trend_following import TrendFollowingStrategy
|
|
from universe import UNIVERSES
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Backtest engine
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def strategy_positions(strategy, data: pd.DataFrame, open_data: pd.DataFrame | None = None,
|
|
bars: dict[str, pd.DataFrame] | None = None) -> pd.DataFrame:
|
|
"""Weights actually held over (close t-1, close t] for each row t.
|
|
|
|
Close mode (``open_data`` is None): row t's weights use closes through t-1
|
|
and trade at the close of t-1 — the same bar the signal saw.
|
|
Live mode (``open_data`` given), matching trader.py:
|
|
* close-price strategies read the session's open as their latest price
|
|
and trade at that session's close;
|
|
* a BarStrategy reads completed bars through t-1 and trades at the close
|
|
of t.
|
|
"""
|
|
if isinstance(strategy, BarStrategy):
|
|
weights = strategy.generate_signals(bars if bars is not None else {"close": data})
|
|
weights = weights.reindex(data.index).fillna(0.0)
|
|
return weights.shift(1).fillna(0.0) if open_data is not None else weights
|
|
if open_data is not None:
|
|
# The strategy's shift(1) on opens gives weights[t] = f(open_{t-1});
|
|
# undo it so the row-t decision uses open_t, then hold it from close t.
|
|
weights = strategy.generate_signals(open_data).shift(-1).fillna(0.0)
|
|
return weights.reindex(data.index).fillna(0.0).shift(1).fillna(0.0)
|
|
return strategy.generate_signals(data).reindex(data.index).fillna(0.0)
|
|
|
|
|
|
def simulate_live_book(held: pd.DataFrame, data: pd.DataFrame, initial_capital: float,
|
|
transaction_cost: float = 0.001, fixed_fee: float = 0.0,
|
|
fee_base: float = 0.0, fee_per_share: float = 0.0,
|
|
min_trade_value: float = 0.0,
|
|
resize_band: float = 0.0) -> tuple[pd.Series, pd.DataFrame]:
|
|
"""Day-by-day book kept the way trader.py keeps it.
|
|
|
|
``held`` row t holds the target weights for (close t-1, close t]. Holdings
|
|
drift with prices; at each close the book trades toward the next row's
|
|
targets: entries and exits always, a held name only once it is at least
|
|
``resize_band`` (relative to its target) away. Orders smaller than
|
|
``max(min_trade_value, 0.1% of equity)`` are skipped unless they close a
|
|
position. Costs are charged on the orders placed.
|
|
|
|
Returns (equity, traded weight per ticker on the row whose holdings changed).
|
|
"""
|
|
targets = held.reindex(index=data.index, columns=data.columns).fillna(0.0).to_numpy()
|
|
prices = data.ffill().to_numpy(dtype=float)
|
|
rets = np.nan_to_num(data.ffill().pct_change(fill_method=None).to_numpy(dtype=float), nan=0.0)
|
|
per_share = np.array([0.0 if str(c).endswith(".HK") else fee_per_share for c in data.columns])
|
|
n, m = targets.shape
|
|
w = np.zeros(m)
|
|
value = float(initial_capital)
|
|
equity = np.empty(n)
|
|
traded = np.zeros((n, m))
|
|
for t in range(n):
|
|
if value <= 0:
|
|
equity[t:] = 0.0
|
|
break
|
|
gross = w * (1.0 + rets[t])
|
|
growth = gross.sum() + (1.0 - w.sum()) # idle cash earns nothing
|
|
value *= growth
|
|
w = gross / growth if growth > 0 else np.zeros(m)
|
|
if t + 1 < n:
|
|
tgt = targets[t + 1]
|
|
new = w.copy()
|
|
entry_exit = (tgt <= 0) | (w <= 0)
|
|
resize = ~entry_exit & (np.abs(tgt - w) >= resize_band * tgt)
|
|
change = (entry_exit | resize) & (np.abs(tgt - w) > 0)
|
|
new[change] = tgt[change]
|
|
if new.sum() > 1.0: # kept winners leave less room
|
|
buys = change & (new > w)
|
|
room = max(1.0 - new[~buys].sum(), 0.0)
|
|
new[buys] *= room / new[buys].sum() if new[buys].sum() > 0 else 0.0
|
|
delta = new - w
|
|
floor = (max(min_trade_value, 0.001 * value) / value) if min_trade_value > 0 else 1e-12
|
|
skip = (np.abs(delta) < floor) & (new > 0)
|
|
delta[skip] = 0.0
|
|
new[skip] = w[skip]
|
|
orders = np.abs(delta) > 1e-12
|
|
if orders.any():
|
|
dollars = np.abs(delta) * value
|
|
cost = dollars.sum() * transaction_cost
|
|
if fixed_fee > 0 or fee_base > 0 or fee_per_share > 0:
|
|
with np.errstate(divide="ignore", invalid="ignore"):
|
|
shares = np.where(prices[t] > 0, dollars / prices[t], 0.0)
|
|
fee = np.maximum(fixed_fee, fee_base + per_share * shares)
|
|
cost += fee[orders].sum()
|
|
if cost >= value: # fees ate the book
|
|
equity[t:] = 0.0
|
|
break
|
|
w = new * value / (value - cost)
|
|
value -= cost
|
|
traded[t + 1] = delta
|
|
equity[t] = value
|
|
return (pd.Series(equity, index=data.index),
|
|
pd.DataFrame(traded, index=data.index, columns=data.columns))
|
|
|
|
|
|
def backtest(
|
|
strategy,
|
|
data: pd.DataFrame,
|
|
initial_capital: float = 100_000,
|
|
transaction_cost: float = 0.001,
|
|
fixed_fee: float = 0.0,
|
|
fee_base: float = 0.0,
|
|
fee_per_share: float = 0.0,
|
|
open_data: pd.DataFrame | None = None,
|
|
min_trade_value: float = 0.0,
|
|
bars: dict[str, pd.DataFrame] | None = None,
|
|
resize_band: float = 0.0,
|
|
) -> pd.Series:
|
|
"""
|
|
Vectorized backtest.
|
|
|
|
Parameters
|
|
----------
|
|
strategy : Strategy
|
|
Any class implementing generate_signals(data) → DataFrame of weights.
|
|
data : pd.DataFrame
|
|
Adjusted close prices, one column per asset.
|
|
initial_capital : float
|
|
Starting portfolio value.
|
|
transaction_cost : float
|
|
One-way cost per unit of turnover (e.g. 0.001 = 10 bps).
|
|
fixed_fee : float
|
|
Floor of the per-trade fee (e.g. 2.0 = $2 minimum per buy/sell).
|
|
With fee_per_share=0 (default), this is also the actual per-trade fee.
|
|
fee_base : float
|
|
Fixed component of a per-share tiered fee schedule. The actual
|
|
per-trade fee is ``max(fixed_fee, fee_base + fee_per_share * shares)``.
|
|
fee_per_share : float
|
|
Per-share variable component of the tiered fee (e.g. 0.009 = $0.009/share).
|
|
With fee_base=1.88 + fee_per_share=0.009 + fixed_fee=2.0 you get an
|
|
IBKR-style schedule: max(2, 1.88 + 0.009 * shares).
|
|
open_data : pd.DataFrame, optional
|
|
Open prices. When provided, the backtest keeps the book the way the
|
|
live trader does: its signal timing (``strategy_positions``), drifting
|
|
holdings, whole orders and ``resize_band`` (``simulate_live_book``).
|
|
min_trade_value : float
|
|
Per-ticker rebalances smaller than ``max(min_trade_value, 0.1% of
|
|
equity)`` are not charged the per-trade fee, mirroring the live
|
|
trader, which never places such orders. 0 (default) charges every
|
|
weight change.
|
|
bars : dict, optional
|
|
``{field: DataFrame}`` panels for a BarStrategy (close/open/high/low/volume).
|
|
|
|
Returns
|
|
-------
|
|
pd.Series
|
|
Daily equity curve.
|
|
"""
|
|
positions = strategy_positions(strategy, data, open_data=open_data, bars=bars)
|
|
if open_data is not None:
|
|
equity, _ = simulate_live_book(
|
|
positions, data, initial_capital, transaction_cost, fixed_fee, fee_base,
|
|
fee_per_share, min_trade_value, resize_band,
|
|
)
|
|
return equity
|
|
|
|
# A missing close carries the last price (zero return), as before.
|
|
daily_returns = data.ffill().pct_change(fill_method=None).fillna(0.0)
|
|
portfolio_returns = (daily_returns * positions).sum(axis=1)
|
|
|
|
# Turnover cost: sum of absolute weight changes each day
|
|
turnover = positions.diff().abs().sum(axis=1).fillna(0.0)
|
|
portfolio_returns -= turnover * transaction_cost
|
|
|
|
# Per-trade fee. Supports both flat ($2/trade) and tiered (IBKR-style)
|
|
# schedules: fee = max(fixed_fee, fee_base + fee_per_share * shares).
|
|
if fixed_fee > 0 or fee_base > 0 or fee_per_share > 0:
|
|
weight_changes = positions.diff().fillna(0.0)
|
|
equity_running = (1 + portfolio_returns).cumprod() * initial_capital
|
|
eq_prev = equity_running.shift(1).fillna(initial_capital)
|
|
if min_trade_value > 0:
|
|
threshold = np.maximum(min_trade_value, 0.001 * eq_prev.abs())
|
|
charged = weight_changes.abs().mul(eq_prev.abs(), axis=0).ge(threshold, axis=0)
|
|
else:
|
|
charged = weight_changes.abs() > 1e-8
|
|
|
|
if fee_per_share > 0:
|
|
# Convert per-ticker weight change into share count traded.
|
|
# dollar_traded[i, t] = |w[i,t] - w[i,t-1]| * equity[t-1]
|
|
# shares_traded[i, t] = dollar_traded / price[i, t]
|
|
dollar_traded = weight_changes.abs().mul(eq_prev, axis=0)
|
|
shares_traded = dollar_traded.div(data).replace(
|
|
[np.inf, -np.inf], 0.0,
|
|
).fillna(0.0)
|
|
# The per-share schedule applies to US-listed shares only.
|
|
per_share = pd.Series(fee_per_share, index=data.columns)
|
|
per_share[[c for c in data.columns if str(c).endswith(".HK")]] = 0.0
|
|
per_trade_fee = (fee_base + shares_traded.mul(per_share, axis=1)).clip(
|
|
lower=fixed_fee,
|
|
)
|
|
per_trade_fee = per_trade_fee.where(charged, 0.0)
|
|
daily_fee = per_trade_fee.sum(axis=1)
|
|
else:
|
|
n_trades = charged.sum(axis=1)
|
|
daily_fee = n_trades * fixed_fee
|
|
|
|
fee_impact = daily_fee / eq_prev
|
|
portfolio_returns -= fee_impact
|
|
|
|
equity = (1 + portfolio_returns).cumprod() * initial_capital
|
|
# A portfolio that loses everything stays at zero; compounding negative
|
|
# equity would otherwise produce meaningless values and metrics.
|
|
ruined = (equity <= 0).cummax()
|
|
if ruined.any():
|
|
equity = equity.where(~ruined, 0.0)
|
|
return equity
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Visualization
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def plot_results(results: pd.DataFrame) -> None:
|
|
"""Two-panel chart: equity curves (top) + drawdowns (bottom)."""
|
|
# Compute drawdowns
|
|
drawdowns = results.div(results.cummax()) - 1
|
|
|
|
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 9), sharex=True,
|
|
gridspec_kw={"height_ratios": [3, 1]})
|
|
|
|
for col in results.columns:
|
|
ax1.plot(results.index, results[col], label=col, linewidth=1.5)
|
|
ax1.set_title("Strategy Comparison — Equity Curves", fontsize=14)
|
|
ax1.set_ylabel("Portfolio Value ($)")
|
|
ax1.legend(loc="upper left")
|
|
ax1.grid(True, alpha=0.3)
|
|
|
|
for col in drawdowns.columns:
|
|
ax2.plot(drawdowns.index, drawdowns[col] * 100, label=col, linewidth=1.0)
|
|
ax2.set_title("Drawdowns")
|
|
ax2.set_ylabel("Drawdown (%)")
|
|
ax2.set_xlabel("Date")
|
|
ax2.grid(True, alpha=0.3)
|
|
|
|
plt.tight_layout()
|
|
plt.show()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Run strategy backtest")
|
|
parser.add_argument(
|
|
"--market", choices=UNIVERSES.keys(), default="us",
|
|
help="Market universe to backtest (default: us)",
|
|
)
|
|
parser.add_argument(
|
|
"--capital", type=float, default=None,
|
|
help="Initial capital (default: 10000)",
|
|
)
|
|
parser.add_argument(
|
|
"--top-n", type=int, default=None,
|
|
help="Number of stocks for selective strategies (default: ~10%% of universe)",
|
|
)
|
|
parser.add_argument(
|
|
"--years", type=int, default=None,
|
|
help="Limit backtest to last N years of data",
|
|
)
|
|
parser.add_argument(
|
|
"--no-plot", action="store_true",
|
|
help="Skip plotting charts",
|
|
)
|
|
parser.add_argument(
|
|
"--fixed-fee", type=float, default=0.0,
|
|
help="Fixed dollar cost per trade, e.g. 2.0 means $2 per buy or sell",
|
|
)
|
|
parser.add_argument(
|
|
"--execution", choices=["close", "open-close"], default="close",
|
|
help="Execution mode: 'close' (default, signal & execute on close) or "
|
|
"'open-close' (signal on morning open, execute at close)",
|
|
)
|
|
parser.add_argument(
|
|
"--attribution", action="store_true",
|
|
help="Run factor attribution after performance metrics",
|
|
)
|
|
parser.add_argument(
|
|
"--attribution-model", choices=["capm", "ff5", "ff5plus", "all"], default="all",
|
|
help="Factor model selection for attribution output",
|
|
)
|
|
parser.add_argument(
|
|
"--attribution-export", default=None,
|
|
help="Directory to export factor attribution CSVs",
|
|
)
|
|
args = parser.parse_args()
|
|
initial_capital = args.capital if args.capital is not None else 10_000
|
|
use_open = args.execution == "open-close"
|
|
|
|
universe = UNIVERSES[args.market]
|
|
tickers = universe["fetch"]()
|
|
benchmark = universe["benchmark"]
|
|
benchmark_label = universe["benchmark_label"]
|
|
|
|
# PIT universe: include all historical index members for US market
|
|
pit_intervals = None
|
|
if args.market == "us":
|
|
pit_intervals = uh.load_sp500_history()
|
|
historical_tickers = uh.all_tickers_ever(pit_intervals)
|
|
all_tickers = sorted(set(tickers + historical_tickers + [benchmark]))
|
|
print(f"--- PIT universe: {len(all_tickers)} tickers (current + historical members) ---")
|
|
else:
|
|
all_tickers = sorted(set(tickers + [benchmark]))
|
|
|
|
result = data_manager.update(args.market, all_tickers, with_open=use_open)
|
|
if use_open:
|
|
data, open_data = result
|
|
else:
|
|
data = result
|
|
open_data = None
|
|
|
|
if args.years:
|
|
cutoff = data.index[-1] - pd.DateOffset(years=args.years)
|
|
data = data[data.index >= cutoff]
|
|
if open_data is not None:
|
|
open_data = open_data[open_data.index >= cutoff]
|
|
print(f"--- Sliced to last {args.years} years: {data.index[0].date()} to {data.index[-1].date()} ---")
|
|
|
|
# Apply PIT mask: NaN out prices for non-member dates
|
|
if pit_intervals is not None:
|
|
print("--- Applying PIT membership mask (survivorship-bias fix) ---")
|
|
data = uh.mask_prices(data, pit_intervals)
|
|
if open_data is not None:
|
|
open_data = uh.mask_prices(open_data, pit_intervals)
|
|
|
|
# Filter tickers to only those with any valid data
|
|
if pit_intervals is not None:
|
|
tickers = [t for t in data.columns if t != benchmark and data[t].notna().any()]
|
|
else:
|
|
tickers = [t for t in tickers if t in data.columns]
|
|
print(f"--- Universe: {len(tickers)} stocks + {benchmark} benchmark ---")
|
|
|
|
top_n = args.top_n if args.top_n else max(5, len(tickers) // 10)
|
|
print(f"--- Selective strategies will pick top {top_n} stocks ---")
|
|
if args.fixed_fee > 0:
|
|
print(f"--- Fixed fee: ${args.fixed_fee:.2f} per trade ---")
|
|
if use_open:
|
|
print(f"--- Execution: open-close (signal on open, execute at close) ---")
|
|
|
|
# Build strategy tuples: (strategy, close_data, open_data_or_None)
|
|
open_tickers = open_data[tickers] if open_data is not None else None
|
|
open_all = open_data if open_data is not None else None
|
|
|
|
strategies = {
|
|
# --- Original strategies ---
|
|
"Buy & Hold (EW)": (BuyAndHoldStrategy(), data[tickers], open_tickers),
|
|
"Momentum": (MomentumStrategy(lookback=252, skip=21, top_n=top_n), data[tickers], open_tickers),
|
|
"Inverse Volatility": (InverseVolatilityStrategy(vol_window=20), data[tickers], open_tickers),
|
|
"Multi-Factor": (MultiFactorStrategy(tickers=tickers, benchmark=benchmark, top_n=top_n), data, open_all),
|
|
# --- New strategies ---
|
|
"Mean Reversion": (MeanReversionStrategy(top_n=top_n), data[tickers], open_tickers),
|
|
"Trend Following": (TrendFollowingStrategy(ma_window=150, momentum_period=126, top_n=top_n), data[tickers], open_tickers),
|
|
"Dual Momentum": (DualMomentumStrategy(top_n=top_n), data[tickers], open_tickers),
|
|
"Momentum+Quality": (MomentumQualityStrategy(momentum_period=252, skip=21, top_n=top_n), data[tickers], open_tickers),
|
|
"Mom+InvVol": (AdaptiveMomentumStrategy(top_n=top_n), data[tickers], open_tickers),
|
|
"Recovery+Mom Top20": (RecoveryMomentumStrategy(top_n=min(20, top_n)), data[tickers], open_tickers),
|
|
"Recovery+Mom Top10": (RecoveryMomentumStrategy(top_n=10), data[tickers], open_tickers),
|
|
}
|
|
|
|
results: dict[str, pd.Series] = {}
|
|
for name, (strategy, strat_data, strat_open) in strategies.items():
|
|
print(f"\nRunning: {name}")
|
|
results[name] = backtest(strategy, strat_data, initial_capital=initial_capital,
|
|
fixed_fee=args.fixed_fee, open_data=strat_open)
|
|
|
|
# Add benchmark (normalized to same initial capital)
|
|
bench = data[benchmark].dropna()
|
|
results[benchmark_label] = (bench / bench.iloc[0]) * initial_capital
|
|
|
|
results_df = pd.DataFrame(results)
|
|
|
|
# --- Performance metrics ---
|
|
for name, equity in results_df.items():
|
|
eq = equity.dropna()
|
|
if len(eq) < 2:
|
|
print(f"\n {name}: insufficient data, skipping")
|
|
continue
|
|
metrics.summary(eq, name=name)
|
|
|
|
if args.attribution:
|
|
summary_df, loadings_df = factor_attribution.attribute_strategies(
|
|
results_df=results_df,
|
|
benchmark_label=benchmark_label,
|
|
benchmark=benchmark,
|
|
price_data=data,
|
|
market=args.market,
|
|
model_selection=args.attribution_model,
|
|
)
|
|
factor_attribution.print_attribution_summary(summary_df)
|
|
if args.attribution_export:
|
|
factor_attribution.export_attribution(summary_df, loadings_df, args.attribution_export)
|
|
print(f"Attribution CSVs written to {args.attribution_export}")
|
|
|
|
# --- Visualization ---
|
|
if not args.no_plot:
|
|
plot_results(results_df.dropna())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|