feat: robust-momentum strategies live; fix live universe, lots and UI backtests

Strategies
- BarStrategy protocol (close/open/high/low/volume bars; row d uses bars
  through d-1) and RobustMomentumStrategy: overnight_recovery_top10 (US),
  lowrisk_momentum_top15 and lowbeta_sharpe_mom_top20 (CN), top-N with
  rank hysteresis, 10-day rebalance, weights drift between rebalances.
- trader: strategy_fields/strategy_bars/bar_session_weights; one signal
  path (_signals_for_sessions) replaces generate_target_weights.

Live fixes
- CSI 300 constituents come from China Securities Index's file; the
  Wikipedia table was 122 names off the real index.
- S&P 500 change log moved to "Historical components of the S&P 500".
- data_manager: tickers missing from a partial Yahoo batch are retried
  alone before being marked unavailable (CEG, GEV, SNDK ... were dropped).
  update() takes fields=[...] and stores high/low/volume panels.
- Whole-lot orders go to the nearest lot within cash (CN books held ~62%
  cash); held names are resized only beyond RESIZE_BAND (10%), so books no
  longer pay a ticket fee to trade daily drift.

Web backtests
- backtest_runner.py replaces the duplicated bridge/precompute code: PIT
  membership in both markets (data/csi300_history.json), stock universe
  limited to index members (ETFs and benchmarks in the shared panel were
  pickable), and the book kept like the live trader (simulate_live_book:
  signal timing, drift, order floor, resize band, fees on orders placed).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-25 19:12:02 +08:00
parent 0e378efa67
commit cc1366b2dc
16 changed files with 16027 additions and 606 deletions

View File

@@ -20,6 +20,7 @@ import os
import urllib.request
from datetime import date, datetime
import numpy as np
import pandas as pd
CACHE_DIR = "data"
@@ -30,34 +31,51 @@ _HEADERS = {"User-Agent": "Mozilla/5.0 (quant-backtest)"}
# Fetch + parse Wikipedia
# ---------------------------------------------------------------------------
def _fetch_sp500_tables() -> tuple[pd.DataFrame, pd.DataFrame]:
"""Return (current_list, changes_log) from the S&P 500 Wikipedia page."""
url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
SP500_LIST_URL = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
# The change log moved off the list page in 2026; the list page is still tried
# first in case it moves back.
SP500_CHANGES_URLS = [
SP500_LIST_URL,
"https://en.wikipedia.org/wiki/Historical_components_of_the_S%26P_500",
]
def _read_tables(url: str) -> list[pd.DataFrame]:
req = urllib.request.Request(url, headers=_HEADERS)
with urllib.request.urlopen(req) as resp:
html = resp.read().decode("utf-8")
tables = pd.read_html(io.StringIO(html))
current = tables[0]
change_tables = [
t for t in tables[1:]
if any("Added" in "_".join(map(str, c if isinstance(c, tuple) else (c,))) for c in t.columns)
]
if not change_tables:
raise ValueError("S&P 500 change log table not found on the Wikipedia page")
changes = change_tables[0]
changes.columns = [
"_".join(c).strip() if isinstance(c, tuple) else c
for c in changes.columns
]
changes.columns = [
c.replace("Effective Date_Effective Date", "Date") for c in changes.columns
]
return current, changes
return pd.read_html(io.StringIO(html))
def _find_changes_table(tables: list[pd.DataFrame]) -> pd.DataFrame | None:
for t in tables:
if any("Added" in "_".join(map(str, c if isinstance(c, tuple) else (c,))) for c in t.columns):
changes = t.copy()
changes.columns = [
"_".join(c).strip() if isinstance(c, tuple) else c
for c in changes.columns
]
changes.columns = [
c.replace("Effective Date_Effective Date", "Date") for c in changes.columns
]
return changes
return None
def _fetch_sp500_tables() -> tuple[pd.DataFrame, pd.DataFrame]:
"""Return (current_list, changes_log) from Wikipedia."""
current = _read_tables(SP500_LIST_URL)[0]
for url in SP500_CHANGES_URLS:
changes = _find_changes_table(_read_tables(url))
if changes is not None:
return current, changes
raise ValueError("S&P 500 change log table not found on Wikipedia")
def _normalize_ticker(t: str) -> str:
"""Yahoo Finance ticker format: BRK.B → BRK-B."""
return str(t).replace(".", "-").strip()
# Some change-log cells carry a trailing " |" separator.
return str(t).split("|")[0].replace(".", "-").strip()
# ---------------------------------------------------------------------------
@@ -243,3 +261,101 @@ def mask_prices(prices: pd.DataFrame,
"""
mask = membership_mask(prices.index, intervals, tickers=list(prices.columns))
return prices.where(mask)
# ---------------------------------------------------------------------------
# CSI 300 (China A-shares)
# ---------------------------------------------------------------------------
#
# data/csi300_history.json holds monthly constituent snapshots
# {"snapshots": {"YYYY-MM-DD": [yahoo tickers], ...}}
# seeded from baostock's query_hs300_stocks (first trading day of each month,
# 2008 onward). A snapshot applies from its date until the next one, so an
# index change is picked up within a month. load_csi300_history() appends a
# snapshot whenever the current constituent list (universe.get_csi300) differs
# from the latest one, which keeps the file current without baostock.
def _csi300_path() -> str:
return os.path.join(CACHE_DIR, "csi300_history.json")
def baostock_to_yahoo(code: str) -> str:
"""'sh.600000' -> '600000.SS', 'sz.000001' -> '000001.SZ'."""
ex, num = code.split(".")
return f"{num}.{'SS' if ex == 'sh' else 'SZ'}"
def load_csi300_history(current: list[str] | None = None) -> dict[str, list[str]]:
"""Monthly CSI 300 snapshots; appends ``current`` (or today's list) if it changed."""
path = _csi300_path()
with open(path) as f:
data = json.load(f)
snaps = data["snapshots"]
if current is None:
try:
from universe import get_csi300
current = get_csi300()
except Exception as exc: # noqa: BLE001 — offline runs use the stored history
print(f"--- WARNING: CSI 300 list unavailable ({exc}); using stored history ---")
current = None
if current:
last = snaps[max(snaps)]
today = str(date.today())
if sorted(current) != sorted(last) and today > max(snaps):
snaps[today] = sorted(current)
with open(path, "w") as f:
json.dump(data, f, indent=0)
print(f"--- CSI 300 constituents changed; stored snapshot {today} ---")
return snaps
def csi300_membership_mask(dates: pd.DatetimeIndex, tickers: list[str],
snapshots: dict[str, list[str]] | None = None) -> pd.DataFrame:
"""Boolean mask (dates x tickers): True while the ticker was in the CSI 300."""
snaps = snapshots if snapshots is not None else load_csi300_history()
keys = sorted(snaps)
col = {t: i for i, t in enumerate(tickers)}
arr = np.zeros((len(dates), len(tickers)), dtype=bool)
for i, k in enumerate(keys):
start = pd.Timestamp(k)
end = pd.Timestamp(keys[i + 1]) if i + 1 < len(keys) else dates[-1] + pd.Timedelta(days=1)
rows = (dates >= start) & (dates < end)
cols = [col[t] for t in snaps[k] if t in col]
if rows.any() and cols:
arr[np.ix_(rows, cols)] = True
mask = pd.DataFrame(arr, index=dates, columns=tickers)
# Tickers never in the index (benchmarks, ETFs) pass through unmasked.
ever = {t for lst in snaps.values() for t in lst}
for t in tickers:
if t not in ever:
mask[t] = True
return mask
def all_csi300_tickers_ever(snapshots: dict[str, list[str]] | None = None,
since: str | None = None) -> list[str]:
snaps = snapshots if snapshots is not None else load_csi300_history()
return sorted({t for k, lst in snaps.items() if since is None or k >= since for t in lst})
# ---------------------------------------------------------------------------
# Market-generic helpers
# ---------------------------------------------------------------------------
def pit_tickers_ever(market: str, since: str | None = None) -> list[str] | None:
"""Every ticker that was an index member (since ``since``), or None when the
market has no membership history."""
if market == "us":
return all_tickers_ever()
if market == "cn":
return all_csi300_tickers_ever(since=since)
return None
def mask_prices_pit(prices: pd.DataFrame, market: str) -> pd.DataFrame:
"""NaN out (date, ticker) pairs outside point-in-time index membership."""
if market == "us":
return mask_prices(prices)
if market == "cn":
return prices.where(csi300_membership_mask(prices.index, list(prices.columns)))
return prices