gahow/quant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: backtest engine fee model, metrics, and strategy fixes

Browse Source 
- main.py: add IBKR-style tiered fee schedule (fee_base + fee_per_share),
  PIT universe support, and open-to-close execution improvements
- metrics.py: add raw_summary helper for JSON-safe metric export
- Misc strategy fixes: deprecation warnings, NaN handling

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-21 20:57:56 +08:00
parent 1f50253d13
commit 149a00c458
7 changed files with 154 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
data/sp500_history.json
View File

File diff suppressed because one or more lines are too long

70
main.py
View File

@@ -7,6 +7,7 @@ import pandas as pd
import data_manager
import factor_attribution
import metrics
import universe_history as uh
from strategies.adaptive_momentum import AdaptiveMomentumStrategy
from strategies.buy_and_hold import BuyAndHoldStrategy
from strategies.dual_momentum import DualMomentumStrategy
@@ -30,6 +31,8 @@ def backtest(
initial_capital: float = 100_000,
transaction_cost: float = 0.001,
fixed_fee: float = 0.0,
fee_base: float = 0.0,
fee_per_share: float = 0.0,
open_data: pd.DataFrame | None = None,
) -> pd.Series:
"""
@@ -46,14 +49,17 @@ def backtest(
transaction_cost : float
One-way cost per unit of turnover (e.g. 0.001 = 10 bps).
fixed_fee : float
Fixed dollar cost per individual trade (each buy or sell).
Floor of the per-trade fee (e.g. 2.0 = $2 minimum per buy/sell).
With fee_per_share=0 (default), this is also the actual per-trade fee.
fee_base : float
Fixed component of a per-share tiered fee schedule. The actual
per-trade fee is ``max(fixed_fee, fee_base + fee_per_share * shares)``.
fee_per_share : float
Per-share variable component of the tiered fee (e.g. 0.009 = $0.009/share).
With fee_base=1.88 + fee_per_share=0.009 + fixed_fee=2.0 you get an
IBKR-style schedule: max(2, 1.88 + 0.009 * shares).
open_data : pd.DataFrame, optional
Open prices. When provided, enables open-to-close execution mode:
- Morning: observe open prices → run strategy → decide weights
- Evening: execute all trades at close prices
Strategies have an internal shift(1) designed for close prices.
Since open prices are observable same-day (before close), we undo
that shift so signals use today's open and execute at today's close.
Open prices. When provided, enables open-to-close execution mode.
Returns
-------
@@ -86,13 +92,32 @@ def backtest(
turnover = positions.diff().abs().sum(axis=1).fillna(0.0)
portfolio_returns -= turnover * transaction_cost
# Fixed per-trade fee: count positions with non-zero weight change
if fixed_fee > 0:
# Per-trade fee. Supports both flat ($2/trade) and tiered (IBKR-style)
# schedules: fee = max(fixed_fee, fee_base + fee_per_share * shares).
if fixed_fee > 0 or fee_base > 0 or fee_per_share > 0:
weight_changes = positions.diff().fillna(0.0)
n_trades = (weight_changes.abs() > 1e-8).sum(axis=1)
# Build running equity to convert dollar fees to return impact
equity_running = (1 + portfolio_returns).cumprod() * initial_capital
fee_impact = (n_trades * fixed_fee) / equity_running.shift(1).fillna(initial_capital)
eq_prev = equity_running.shift(1).fillna(initial_capital)
if fee_per_share > 0:
# Convert per-ticker weight change into share count traded.
# dollar_traded[i, t] = |w[i,t] - w[i,t-1]| * equity[t-1]
# shares_traded[i, t] = dollar_traded / price[i, t]
dollar_traded = weight_changes.abs().mul(eq_prev, axis=0)
shares_traded = dollar_traded.div(data).replace(
[np.inf, -np.inf], 0.0,
).fillna(0.0)
per_trade_fee = (fee_base + fee_per_share * shares_traded).clip(
lower=fixed_fee,
)
trade_mask = weight_changes.abs() > 1e-8
per_trade_fee = per_trade_fee.where(trade_mask, 0.0)
daily_fee = per_trade_fee.sum(axis=1)
else:
n_trades = (weight_changes.abs() > 1e-8).sum(axis=1)
daily_fee = n_trades * fixed_fee
fee_impact = daily_fee / eq_prev
portfolio_returns -= fee_impact
equity = (1 + portfolio_returns).cumprod() * initial_capital
@@ -184,6 +209,15 @@ def main() -> None:
tickers = universe["fetch"]()
benchmark = universe["benchmark"]
benchmark_label = universe["benchmark_label"]
# PIT universe: include all historical index members for US market
pit_intervals = None
if args.market == "us":
pit_intervals = uh.load_sp500_history()
historical_tickers = uh.all_tickers_ever(pit_intervals)
all_tickers = sorted(set(tickers + historical_tickers + [benchmark]))
print(f"--- PIT universe: {len(all_tickers)} tickers (current + historical members) ---")
else:
all_tickers = sorted(set(tickers + [benchmark]))
result = data_manager.update(args.market, all_tickers, with_open=use_open)
@@ -200,7 +234,17 @@ def main() -> None:
open_data = open_data[open_data.index >= cutoff]
print(f"--- Sliced to last {args.years} years: {data.index[0].date()} to {data.index[-1].date()} ---")
# Filter tickers to only those in the data
# Apply PIT mask: NaN out prices for non-member dates
if pit_intervals is not None:
print("--- Applying PIT membership mask (survivorship-bias fix) ---")
data = uh.mask_prices(data, pit_intervals)
if open_data is not None:
open_data = uh.mask_prices(open_data, pit_intervals)
# Filter tickers to only those with any valid data
if pit_intervals is not None:
tickers = [t for t in data.columns if t != benchmark and data[t].notna().any()]
else:
tickers = [t for t in tickers if t in data.columns]
print(f"--- Universe: {len(tickers)} stocks + {benchmark} benchmark ---")

15
metrics.py
View File

@@ -52,6 +52,21 @@ def win_rate(returns: pd.Series) -> float:
return (active > 0).sum() / len(active)
def raw_summary(equity: pd.Series) -> dict:
"""Return numeric metrics suitable for JSON serialization."""
returns = equity.pct_change().dropna()
return {
"totalReturn": float(total_return(equity)),
"annualizedReturn": float(annualized_return(equity)),
"annualizedVolatility": float(annualized_volatility(returns)),
"sharpeRatio": float(sharpe_ratio(returns)),
"sortinoRatio": float(sortino_ratio(returns)),
"maxDrawdown": float(max_drawdown(equity)),
"calmarRatio": float(calmar_ratio(equity)),
"winRate": float(win_rate(returns)),
}
def summary(equity: pd.Series, name: str = "Strategy") -> dict:
returns = equity.pct_change().dropna()
metrics = {

2
strategies/buy_and_hold.py
View File

@@ -12,4 +12,4 @@ class BuyAndHoldStrategy(Strategy):
"""
tickers = data.columns
weights = pd.DataFrame(1 / len(tickers), index=data.index, columns=tickers)
return weights
return weights.shift(1).fillna(0.0)

6
strategies/dual_momentum.py
View File

@@ -30,9 +30,9 @@ class DualMomentumStrategy(Strategy):
all_positive = (short_mom > 0) & (med_mom > 0) & (long_mom > 0)
# Composite score: average percentile rank across timeframes
short_rank = short_mom.rank(axis=1, pct=True, na_option="bottom")
med_rank = med_mom.rank(axis=1, pct=True, na_option="bottom")
long_rank = long_mom.rank(axis=1, pct=True, na_option="bottom")
short_rank = short_mom.rank(axis=1, pct=True, na_option="keep")
med_rank = med_mom.rank(axis=1, pct=True, na_option="keep")
long_rank = long_mom.rank(axis=1, pct=True, na_option="keep")
composite = (short_rank + med_rank + long_rank) / 3
# Only consider stocks passing absolute filter

119
strategies/ensemble_alpha.py
View File

@@ -84,17 +84,6 @@ class EnsembleAlphaStrategy(Strategy):
row_sums = raw.sum(axis=1).replace(0, np.nan)
signals = raw.div(row_sums, axis=0).fillna(0.0)
# === Tail-risk protection ===
if self.tail_protection:
# Portfolio equity proxy: equal-weight market return
mkt_ret = ret.mean(axis=1)
mkt_eq = (1 + mkt_ret).cumprod()
mkt_dd = mkt_eq / mkt_eq.cummax() - 1
in_tail = mkt_dd < self.tail_threshold
scale = pd.Series(1.0, index=data.index)
scale[in_tail] = self.tail_scale
signals = signals.mul(scale, axis=0)
# === Monthly rebalance ===
warmup = 252
rebal_mask = pd.Series(False, index=data.index)
@@ -105,6 +94,20 @@ class EnsembleAlphaStrategy(Strategy):
signals = signals.ffill().fillna(0.0)
signals.iloc[:warmup] = 0.0
# === Tail-risk protection (rebal-gated; NaN-safe market mean) ===
# Apply AFTER ffill so the scale changes only at rebal points,
# otherwise daily flips force half-out/half-in trades that burn
# the account through fixed per-trade fees.
if self.tail_protection:
mkt_ret = ret.mean(axis=1, skipna=True)
mkt_eq = (1 + mkt_ret.fillna(0.0)).cumprod()
mkt_dd = mkt_eq / mkt_eq.cummax() - 1
in_tail = mkt_dd < self.tail_threshold
scale_raw = pd.Series(1.0, index=data.index)
scale_raw[in_tail] = self.tail_scale
scale = scale_raw.where(rebal_mask, np.nan).ffill().fillna(1.0)
signals = signals.mul(scale, axis=0)
return signals.shift(1).fillna(0.0)
@@ -162,16 +165,6 @@ class EnhancedFactorComboStrategy(Strategy):
row_sums = raw.sum(axis=1).replace(0, np.nan)
signals = raw.div(row_sums, axis=0).fillna(0.0)
# Tail protection
if self.tail_protection:
mkt_ret = ret.mean(axis=1)
mkt_eq = (1 + mkt_ret).cumprod()
mkt_dd = mkt_eq / mkt_eq.cummax() - 1
in_tail = mkt_dd < -0.15
scale = pd.Series(1.0, index=data.index)
scale[in_tail] = 0.5
signals = signals.mul(scale, axis=0)
# Monthly rebalance
warmup = 252
rebal_mask = pd.Series(False, index=data.index)
@@ -182,6 +175,17 @@ class EnhancedFactorComboStrategy(Strategy):
signals = signals.ffill().fillna(0.0)
signals.iloc[:warmup] = 0.0
# Tail protection (rebal-gated to avoid daily-flip turnover)
if self.tail_protection:
mkt_ret = ret.mean(axis=1, skipna=True)
mkt_eq = (1 + mkt_ret.fillna(0.0)).cumprod()
mkt_dd = mkt_eq / mkt_eq.cummax() - 1
in_tail = mkt_dd < -0.15
scale_raw = pd.Series(1.0, index=data.index)
scale_raw[in_tail] = 0.5
scale = scale_raw.where(rebal_mask, np.nan).ffill().fillna(1.0)
signals = signals.mul(scale, axis=0)
return signals.shift(1).fillna(0.0)
@@ -230,31 +234,41 @@ class RiskManagedEnsembleStrategy(Strategy):
# Step 1: Get raw signals from the ensemble (already shifted by 1)
raw = self.ensemble.generate_signals(data)
# Step 2: Compute MARKET returns (equal-weight of all stocks)
daily_rets = data.pct_change().fillna(0.0)
mkt_rets = daily_rets.mean(axis=1)
# Step 2: Compute MARKET returns over valid (non-masked) columns.
# `daily_rets` keeps NaN for PIT-masked tickers so they don't dilute
# the cross-sectional mean to ~0.
daily_rets = data.pct_change()
mkt_rets = daily_rets.mean(axis=1, skipna=True)
# Step 3: Market drawdown dampener
mkt_eq = (1 + mkt_rets).cumprod()
mkt_dd = mkt_eq / mkt_eq.cummax() - 1 # always ≤ 0
# Linear: at DD=0 → 1.0, at DD=-dd_denom → dd_floor
dd_scale = (1.0 + mkt_dd / self.dd_denom).clip(lower=self.dd_floor, upper=1.0)
dd_scale_lagged = dd_scale.shift(1).fillna(1.0) # PIT
mkt_eq = (1 + mkt_rets.fillna(0.0)).cumprod()
mkt_dd = mkt_eq / mkt_eq.cummax() - 1
dd_scale_raw = (1.0 + mkt_dd / self.dd_denom).clip(
lower=self.dd_floor, upper=1.0,
)
# Step 4: Vol spike guard (uses portfolio's own vol for specificity)
# Step 4: Vol spike guard from portfolio returns (NaN-aware sum)
if self.vol_spike_guard:
port_rets = (raw * daily_rets).sum(axis=1)
port_rets = (raw * daily_rets).sum(axis=1, min_count=1).fillna(0.0)
short_vol = port_rets.rolling(self.vol_spike_window, min_periods=5).std() * np.sqrt(252)
vol_90th = short_vol.rolling(self.vol_spike_lookback, min_periods=126).quantile(0.90)
in_spike = short_vol > vol_90th
vol_scale = pd.Series(1.0, index=data.index)
vol_scale[in_spike] = self.vol_spike_floor
vol_scale_lagged = vol_scale.shift(1).fillna(1.0) # PIT
vol_scale_raw = pd.Series(1.0, index=data.index)
vol_scale_raw[in_spike] = self.vol_spike_floor
else:
vol_scale_lagged = 1.0
vol_scale_raw = pd.Series(1.0, index=data.index)
# Step 5: Combined scaling, sampled at the inner ensemble's rebal
# cadence so we don't trade in/out daily (which would incur huge
# fixed-fee costs).
combined = (dd_scale_raw * vol_scale_raw).shift(1).fillna(1.0)
rebal_freq = getattr(self.ensemble, "rebal_freq", 21)
warmup = 252
rebal_mask = pd.Series(False, index=data.index)
rebal_indices = list(range(warmup, len(data), rebal_freq))
rebal_mask.iloc[rebal_indices] = True
final_scale = combined.where(rebal_mask, np.nan).ffill().fillna(1.0)
# Step 5: Combined scaling
final_scale = dd_scale_lagged * vol_scale_lagged
return raw.mul(final_scale, axis=0)
@@ -342,26 +356,31 @@ class SharpeBoostedEnsembleStrategy(Strategy):
signals = signals.shift(1).fillna(0.0) # PIT: 1-day execution lag
# === Asymmetric vol scaling ===
# Only reduce exposure when vol is high AND returns are negative
# High vol + positive returns = riding a trend, don't cut
daily_rets = data.pct_change().fillna(0.0)
port_rets = (signals * daily_rets).sum(axis=1)
# NB: scales are RE-EVALUATED only on rebalance days. Daily flips of
# asym/dd scales would force half-in/half-out trades each session,
# burning the account through fixed per-trade fees ($2 US / $5 CN).
# Use cross-sectional mean of non-masked returns so PIT-masked
# NaN→0 fills don't dilute the market signal.
daily_rets = data.pct_change()
port_rets = (signals * daily_rets).sum(axis=1, min_count=1).fillna(0.0)
short_vol = port_rets.rolling(20, min_periods=10).std() * np.sqrt(252)
vol_median = short_vol.rolling(252, min_periods=126).median()
recent_ret = port_rets.rolling(20, min_periods=10).sum()
high_vol_neg = (short_vol > vol_median * 1.5) & (recent_ret < 0)
asym_scale = pd.Series(1.0, index=data.index)
asym_scale[high_vol_neg] = self.asym_vol_floor
signals = signals.mul(asym_scale.shift(1).fillna(1.0), axis=0) # PIT
asym_scale_raw = pd.Series(1.0, index=data.index)
asym_scale_raw[high_vol_neg] = self.asym_vol_floor
# === Light market-DD dampener ===
# Uses market (not strategy) drawdown to avoid negative feedback loop
mkt_rets = daily_rets.mean(axis=1)
mkt_eq = (1 + mkt_rets).cumprod()
# === Market-DD dampener (rebal-gated, NaN-aware market mean) ===
mkt_rets = daily_rets.mean(axis=1, skipna=True)
mkt_eq = (1 + mkt_rets.fillna(0.0)).cumprod()
mkt_dd = mkt_eq / mkt_eq.cummax() - 1
dd_scale = (1.0 + mkt_dd / self.dd_denom).clip(
dd_scale_raw = (1.0 + mkt_dd / self.dd_denom).clip(
lower=self.dd_floor, upper=1.0
)
signals = signals.mul(dd_scale.shift(1).fillna(1.0), axis=0) # PIT
# Sample scales at rebal points only, then step-hold between rebals.
combined = (asym_scale_raw * dd_scale_raw).shift(1).fillna(1.0)
rebal_scale = combined.where(rebal_mask, np.nan).ffill().fillna(1.0)
signals = signals.mul(rebal_scale, axis=0)
return signals

10
strategies/trend_rider_v6.py
View File

@@ -89,11 +89,13 @@ class TrendRiderV6(TrendRiderV5):
def _resolve_universe(self, prices: pd.DataFrame) -> list[str]:
if self.stock_universe is not None:
return [s for s in self.stock_universe if s in prices.columns]
# Heuristic: stocks are columns NOT in our known ETF/leveraged set
non_stock = (set(self.core_equity)
| set(self.leveraged_equity)
# Heuristic: stocks are columns NOT in our known ETF/leveraged set.
# We inherit V3's risk_on (e.g. TQQQ/UPRO) and risk_off (GLD/DBC),
# plus V6's risk_off_basket + moderate_anchor + signal + overlay sym.
non_stock = (set(self.risk_on)
| set(self.risk_off)
| {self.signal, *self.risk_off_basket, self.moderate_anchor})
| {self.signal, self.moderate_anchor,
self.leverage_overlay_symbol, *self.risk_off_basket})
return [c for c in prices.columns if c not in non_stock]
def _stock_top_n_weights(self, prices: pd.DataFrame, universe: list[str]) -> pd.DataFrame: