gahow/quant-trading
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(report): correct periodic returns, injection metrics and live view

Browse Source 
- Monthly/yearly returns chain from the previous period's close (the first
  trading day of every period was dropped; up to ~9pp/yr error).
- With cash injections, metrics/drawdowns/periodic returns are time-weighted
  on the no-injection curve; capitalFlows reports invested/profit separately.
- Benchmarks cover exactly the strategies' dates.
- CAGR uses calendar time, so gaps in live records do not inflate it.
- Backtests skip the per-trade fee for rebalances below the live minimum
  order and keep a ruined portfolio at zero.
- live: exact strategy match, ineligible strategies hidden, currency,
  startDate, same-window benchmark, stale-price fallback, equity anchored at
  the initial capital.
- PIT membership falls back to the cached history when the Wikipedia change
  log cannot be parsed (the table is no longer on the page).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-09-25 01:16:33 +08:00
parent 11f4dd95dd
commit 1697bc1641
5 changed files with 333 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

301
bridge.py
View File

@@ -25,12 +25,21 @@ from main import backtest
from research.permanent_yearly import equity_with_cashflows
from trader import (
ETF_STRATEGY_UNIVERSES,
LIVE_EXCLUDED_STRATEGIES,
MARKET_FEE_TIERED,
STRATEGY_REGISTRY,
strategy_universe,
convert_quote_currency,
price_request,
strategy_supports_market,
strategy_data_market,
load_state,
filter_tradable_tickers,
)
MARKET_CURRENCY = {"us": "USD", "cn": "CNY"}
# Rebalances below this size are not charged the per-trade fee in backtests,
# matching the live trader's minimum order size.
MIN_TRADE_VALUE = 50.0
from universe import UNIVERSES
@@ -567,27 +576,37 @@ def compute_rebalance_win_rate(equity: pd.Series, trade_events: list[dict]) -> f
def compute_periodic_returns(equity: pd.Series) -> tuple[dict, dict]:
"""Compute monthly and yearly returns from an equity curve."""
monthly = {}
yearly = {}
"""Monthly and yearly returns from an equity curve.
# Monthly returns
for (year, month), group in equity.groupby([equity.index.year, equity.index.month]):
if len(group) < 2:
continue
ret = group.iloc[-1] / group.iloc[0] - 1
monthly[f"{year}-{month:02d}"] = round(float(ret), 4)
# Yearly returns
for year, group in equity.groupby(equity.index.year):
if len(group) < 2:
continue
ret = group.iloc[-1] / group.iloc[0] - 1
yearly[str(year)] = round(float(ret), 4)
Each period is measured from the previous period's last value (the first
period from the curve's first value), so the first trading day's move of
every period is included.
"""
def by_period(keys) -> dict:
ends = equity.groupby(keys).last()
out = {}
prev = float(equity.iloc[0])
for key, end in ends.items():
if prev > 0:
out[key] = round(float(end) / prev - 1, 4)
prev = float(end)
return out
idx = equity.index
monthly = by_period([f"{d.year}-{d.month:02d}" for d in idx])
yearly = by_period([str(d.year) for d in idx])
return monthly, yearly
def benchmark_curve(prices: pd.Series, dates: pd.DatetimeIndex, capital: float) -> list[dict]:
"""Benchmark value on ``dates``, rebased to ``capital`` on the first date it has a price."""
series = prices.reindex(dates).ffill().dropna()
if series.empty:
return []
values = series / series.iloc[0] * capital
return [{"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)} for d, v in values.items()]
def build_injection_schedule(equity: pd.Series, schedule: dict) -> tuple[pd.Series, list[dict]]:
"""Build a contributions Series from injection schedule config.
@@ -772,17 +791,17 @@ def cmd_backtest(args: argparse.Namespace) -> None:
for name in stock_strategies:
_log(f" Running: {name}")
strategy_fn = STRATEGY_REGISTRY[name]
# Derive top_n from strategy metadata (respects per-strategy defaults)
meta = STRATEGY_META.get(name, {})
top_n = int(meta.get("params", {}).get("top_n", {}).get("default", 10))
strategy = strategy_fn(top_n=top_n)
# Registry defaults are the single source of strategy parameters,
# shared with the live trader.
strategy = STRATEGY_REGISTRY[name]()
equity = backtest(
strategy, data[tickers],
initial_capital=capital,
transaction_cost=tx_cost,
fixed_fee=fixed_fee,
min_trade_value=MIN_TRADE_VALUE,
**MARKET_FEE_TIERED.get(market, {}),
)
# Get weight signals for trade extraction
@@ -795,14 +814,8 @@ def cmd_backtest(args: argparse.Namespace) -> None:
)
results.append(result_entry)
# Benchmark
bench = data[benchmark_ticker].dropna()
if years:
cutoff = bench.index[-1] - pd.DateOffset(years=years)
bench = bench[bench.index >= cutoff]
bench_equity = (bench / bench.iloc[0]) * capital
bench_curve = [{"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)}
for d, v in bench_equity.items()]
# Benchmark over exactly the strategies' dates
bench_curve = benchmark_curve(data[benchmark_ticker], data.index, capital)
else:
benchmark_label = "SPY"
bench_curve = []
@@ -816,16 +829,16 @@ def cmd_backtest(args: argparse.Namespace) -> None:
all_etf = sorted(set(etf_tickers + ["SPY"]))
with _redirect_stdout_to_stderr():
etf_data = data_manager.update("etfs", all_etf, with_open=False)
etf_data = data_manager.update("etfs", price_request(market, all_etf), with_open=False)
if isinstance(etf_data, tuple):
etf_data = etf_data[0]
etf_data = convert_quote_currency(etf_data)
if years:
cutoff = etf_data.index[-1] - pd.DateOffset(years=years)
etf_data = etf_data[etf_data.index >= cutoff]
strategy_fn = STRATEGY_REGISTRY[name]
strategy = strategy_fn()
strategy = STRATEGY_REGISTRY[name]()
tradable = filter_tradable_tickers(etf_data, etf_tickers)
equity = backtest(
@@ -833,6 +846,8 @@ def cmd_backtest(args: argparse.Namespace) -> None:
initial_capital=capital,
transaction_cost=tx_cost,
fixed_fee=fixed_fee,
min_trade_value=MIN_TRADE_VALUE,
**MARKET_FEE_TIERED.get(market, {}),
)
weights = strategy.generate_signals(etf_data[tradable])
@@ -855,10 +870,7 @@ def cmd_backtest(args: argparse.Namespace) -> None:
cutoff = spy_data.index[-1] - pd.DateOffset(years=years)
spy_data = spy_data[spy_data.index >= cutoff]
if "SPY" in spy_data.columns:
bench = spy_data["SPY"].dropna()
bench_equity = (bench / bench.iloc[0]) * capital
bench_curve = [{"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)}
for d, v in bench_equity.items()]
bench_curve = benchmark_curve(spy_data["SPY"], spy_data.index, capital)
output = {
"meta": {
@@ -888,7 +900,13 @@ def _build_strategy_result(
injection_schedule: dict | None,
initial_capital: float,
) -> dict:
"""Build a strategy result dict with optional injection applied."""
"""Build a strategy result dict with optional injection applied.
Return and risk metrics, drawdowns and periodic returns are time-weighted
(computed on the curve without injections), so injected cash never counts
as performance. ``capitalFlows`` reports the money view: total invested,
final value and profit of the injected curve.
"""
injection_events = []
equity_with_inj = None
@@ -897,8 +915,10 @@ def _build_strategy_result(
contributions, injection_events = build_injection_schedule(equity, injection_schedule)
equity_with_inj = equity_with_cashflows(returns, contributions, initial_capital)
# Use injected equity for metrics/curves if available
# Curve shown to the user: the money curve when cash is injected.
final_equity = equity_with_inj if equity_with_inj is not None else equity
invested = initial_capital + sum(ev["amount"] for ev in injection_events)
final_value = float(final_equity.iloc[-1])
# Equity curve
curve = [{"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)}
@@ -910,17 +930,14 @@ def _build_strategy_result(
base_curve = [{"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)}
for d, v in equity.items()]
# Metrics
strat_metrics = metrics.raw_summary(final_equity)
# Time-weighted metrics on the curve without injections
strat_metrics = metrics.raw_summary(equity)
strat_metrics["rebalanceWinRate"] = compute_rebalance_win_rate(
final_equity, trade_events,
equity, trade_events,
)
# Drawdown periods
drawdowns = compute_drawdown_periods(final_equity)
# Monthly/yearly returns
monthly_ret, yearly_ret = compute_periodic_returns(final_equity)
drawdowns = compute_drawdown_periods(equity)
monthly_ret, yearly_ret = compute_periodic_returns(equity)
meta = STRATEGY_META.get(name, {"label": name, "category": "other"})
@@ -933,6 +950,13 @@ def _build_strategy_result(
"drawdowns": drawdowns,
"trades": trade_events,
"injections": injection_events,
"capitalFlows": {
"invested": round(invested, 2),
"finalValue": round(final_value, 2),
"profit": round(final_value - invested, 2),
"returnOnInvested": final_value / invested - 1 if invested else None,
},
"ruined": bool((equity <= 0).any()),
"monthlyReturns": monthly_ret,
"yearlyReturns": yearly_ret,
}
@@ -942,81 +966,81 @@ def _build_strategy_result(
# Subcommand: live
# ---------------------------------------------------------------------------
def _load_latest_prices(markets_needed: set[str]) -> dict[str, dict]:
"""Pull the latest close + as-of date for each cached market panel.
LIVE_BENCHMARK = {"us": ("SPY", "SPY"), "cn": ("510300.SS", "CSI 300 ETF 510300")}
Returns ``{market: {"asof": str|None, "prices": {ticker: float}}}``.
We read the on-disk CSVs directly so this stays fast even when called
against a freshly initialized live state with no network.
def _load_panels(markets_needed: set[str]) -> dict[str, pd.DataFrame | None]:
"""Load each cached close panel once (forward-filled, HK tickers in USD).
Reads the on-disk CSVs directly so the live view stays fast and works
without network access.
"""
import data_manager # local import to avoid heavyweight import at module load
out: dict[str, dict] = {}
panels: dict[str, pd.DataFrame | None] = {}
for mk in markets_needed:
try:
df = data_manager.load(mk)
except Exception as e: # noqa: BLE001 — never fatal
_log(f" load_latest_prices: cannot load {mk}: {e}")
out[mk] = {"asof": None, "prices": {}}
continue
if df is None or df.empty:
out[mk] = {"asof": None, "prices": {}}
continue
last_row = df.iloc[-1]
out[mk] = {
"asof": df.index[-1].strftime("%Y-%m-%d"),
"prices": {col: float(v) for col, v in last_row.items()
if v is not None and not (isinstance(v, float) and np.isnan(v))},
}
_log(f" load panel {mk}: {e}")
df = None
panels[mk] = convert_quote_currency(df) if df is not None and not df.empty else None
return panels
def _latest_prices(panel: pd.DataFrame | None) -> dict:
if panel is None:
return {"asof": None, "prices": {}}
last_row = panel.iloc[-1]
return {
"asof": panel.index[-1].strftime("%Y-%m-%d"),
"prices": {col: float(v) for col, v in last_row.items() if pd.notna(v)},
}
def _last_recorded_prices(state: dict) -> dict[str, float]:
"""Most recent price recorded in the daily log for each ticker."""
out: dict[str, float] = {}
for entry in reversed(state.get("daily_log", []) or []):
for ticker, pos in (entry.get("holdings") or {}).items():
if ticker not in out and pos.get("price"):
out[ticker] = float(pos["price"])
return out
def _strategy_data_market_safe(strategy_name: str, default: str) -> str:
"""Return the price-data market a strategy reads from.
Mirrors trader.strategy_data_market without importing trader (to keep
bridge import-cheap when the CLI is hit). Tactical-allocation strategies
trade ETFs, so they pull from the 'etfs' panel; stock pickers stay on
their nominal market.
"""
try:
from trader import ETF_STRATEGY_UNIVERSES # noqa: WPS433
except Exception:
return default
base = strategy_name.removeprefix("sim_")
return "etfs" if base in ETF_STRATEGY_UNIVERSES else default
def cmd_live(args: argparse.Namespace) -> None:
"""Read live trader state files and output normalized JSON.
Each strategy's record contains:
- Current cash, holdings, marked-to-market value of each holding (using
the latest cached close price), and total assets.
- Daily equity curve from the trader's persisted state.
- Current cash, holdings, marked-to-market value of each holding (latest
cached price, falling back to the last recorded price), and total assets.
- Daily equity curve starting from the initial capital, plus the market
benchmark rebased to the same capital over the same dates.
- Full trade log (one entry per ticker per fill) + day-aggregated view.
- Metrics, drawdowns, monthly/yearly returns.
"""
market = args.market
pattern = os.path.join("data", f"trader_{market}_*.json")
state_files = sorted(glob(pattern))
prefix = f"trader_{market}_"
state_files = sorted(glob(os.path.join("data", f"{prefix}*.json")))
wanted = args.strategy
if args.strategy:
state_files = [f for f in state_files if args.strategy in f]
_log(f"Found {len(state_files)} state files for market={market}")
# Pre-load price panels once. Tactical/ETF strategies read from 'etfs',
# everything else from the nominal market.
markets_needed = {market, "etfs"}
latest_prices = _load_latest_prices(markets_needed)
panels = _load_panels({market, "etfs"})
bench_ticker, bench_label = LIVE_BENCHMARK[market]
bench_panel = panels.get(market)
bench_prices = (
bench_panel[bench_ticker]
if bench_panel is not None and bench_ticker in bench_panel.columns
else pd.Series(dtype=float)
)
results = []
for path in state_files:
basename = os.path.basename(path)
parts = basename.replace(".json", "").split("_", 2)
if len(parts) < 3:
strategy_name = os.path.basename(path)[len(prefix):-len(".json")]
if strategy_name.startswith("sim_"):
continue
if wanted and strategy_name != wanted:
continue
if strategy_name in LIVE_EXCLUDED_STRATEGIES or not strategy_supports_market(strategy_name, market):
_log(f" Skipping {strategy_name}: not a live strategy in the {market.upper()} book")
continue
strategy_name = parts[2]
try:
with open(path) as f:
@@ -1025,19 +1049,22 @@ def cmd_live(args: argparse.Namespace) -> None:
_log(f" Skipping {path}: {e}")
continue
data_market = _strategy_data_market_safe(strategy_name, market)
price_book = latest_prices.get(data_market, {"asof": None, "prices": {}})
data_market = strategy_data_market(market, strategy_name)
price_book = _latest_prices(panels.get(data_market))
recorded = _last_recorded_prices(state)
holdings_raw = state.get("holdings", {}) or {}
cash = float(state.get("cash", 0) or 0)
# Per-holding marked-to-market table
holdings_detail = []
market_value = 0.0
for ticker, shares in holdings_raw.items():
shares_f = float(shares)
price = price_book["prices"].get(ticker)
value = shares_f * float(price) if price is not None else None
stale = price is None
if stale:
price = recorded.get(ticker)
value = shares_f * price if price is not None else None
if value is not None:
market_value += value
holdings_detail.append({
@@ -1045,28 +1072,28 @@ def cmd_live(args: argparse.Namespace) -> None:
"shares": shares_f,
"lastPrice": float(price) if price is not None else None,
"marketValue": round(value, 2) if value is not None else None,
"priceStale": stale,
})
holdings_detail.sort(
key=lambda h: (-(h["marketValue"] or 0.0), h["ticker"]),
)
holdings_detail.sort(key=lambda h: (-(h["marketValue"] or 0.0), h["ticker"]))
# Equity curve from persisted state
initial_capital = float(state.get("initial_capital", 0) or 0)
daily_equity = state.get("daily_equity", {}) or {}
dates = sorted(daily_equity.keys())
equity_curve = [
{"date": d, "value": round(float(daily_equity[d]), 2)}
for d in dates
]
equity_series = pd.Series(
{pd.Timestamp(d): float(daily_equity[d]) for d in dates}
).sort_index() if dates else pd.Series(dtype=float)
{pd.Timestamp(d): float(daily_equity[d]) for d in dates}, dtype=float,
).sort_index()
if len(equity_series) and initial_capital:
# Anchor at the initial capital one day before the first session so
# the first day's P&L is part of every metric.
anchor = equity_series.index[0] - pd.Timedelta(days=1)
equity_series = pd.concat([pd.Series({anchor: initial_capital}), equity_series])
equity_curve = [
{"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)}
for d, v in equity_series.items()
]
strat_metrics = (
metrics.raw_summary(equity_series) if len(equity_series) >= 2 else None
)
strat_metrics = metrics.raw_summary(equity_series) if len(equity_series) >= 2 else None
# Trade-by-day aggregation (kept for backwards compatibility) plus
# full granular log.
trade_log_raw = state.get("trade_log", []) or []
trade_by_date: dict[str, dict] = {}
trade_log = []
@@ -1097,29 +1124,24 @@ def cmd_live(args: argparse.Namespace) -> None:
agg["sellTickers"].append(entry["ticker"])
trades = [{"date": d, **v} for d, v in sorted(trade_by_date.items())]
drawdowns = (
compute_drawdown_periods(equity_series) if len(equity_series) >= 2 else []
)
drawdowns = compute_drawdown_periods(equity_series) if len(equity_series) >= 2 else []
monthly_ret, yearly_ret = (
compute_periodic_returns(equity_series)
if len(equity_series) >= 2 else ({}, {})
compute_periodic_returns(equity_series) if len(equity_series) >= 2 else ({}, {})
)
meta = STRATEGY_META.get(
strategy_name, {"label": strategy_name, "category": "other"},
)
meta = STRATEGY_META.get(strategy_name, {"label": strategy_name, "category": "other"})
# Current value: prefer live MTM; fall back to last persisted equity.
live_value = cash + market_value if holdings_detail else cash
if not holdings_detail and equity_curve:
live_value = equity_curve[-1]["value"]
# Current value: live mark-to-market of holdings plus cash.
live_value = cash + market_value
if not holdings_detail and dates:
live_value = float(daily_equity[dates[-1]])
initial_capital = float(state.get("initial_capital", 0) or 0)
unrealized_pnl = (
live_value - initial_capital if initial_capital else 0.0
)
unrealized_pnl_pct = (
unrealized_pnl / initial_capital if initial_capital else 0.0
unrealized_pnl = live_value - initial_capital if initial_capital else 0.0
unrealized_pnl_pct = unrealized_pnl / initial_capital if initial_capital else 0.0
bench_curve = (
benchmark_curve(bench_prices, pd.DatetimeIndex(equity_series.index[1:]), initial_capital)
if len(equity_series) >= 2 and len(bench_prices) else []
)
results.append({
@@ -1127,7 +1149,9 @@ def cmd_live(args: argparse.Namespace) -> None:
"label": meta.get("label", strategy_name),
"category": meta.get("category", "other"),
"market": market,
"currency": MARKET_CURRENCY.get(market, "USD"),
"initialCapital": initial_capital,
"startDate": dates[0] if dates else None,
"currentValue": round(live_value, 2),
"cash": round(cash, 2),
"marketValue": round(market_value, 2),
@@ -1137,6 +1161,7 @@ def cmd_live(args: argparse.Namespace) -> None:
"holdingsDetail": holdings_detail,
"priceAsOf": price_book["asof"],
"equityCurve": equity_curve,
"benchmark": {"name": bench_label, "equityCurve": bench_curve},
"metrics": strat_metrics,
"drawdowns": drawdowns,
"trades": trades,