feat: robust-momentum strategies live; fix live universe, lots and UI backtests
Strategies - BarStrategy protocol (close/open/high/low/volume bars; row d uses bars through d-1) and RobustMomentumStrategy: overnight_recovery_top10 (US), lowrisk_momentum_top15 and lowbeta_sharpe_mom_top20 (CN), top-N with rank hysteresis, 10-day rebalance, weights drift between rebalances. - trader: strategy_fields/strategy_bars/bar_session_weights; one signal path (_signals_for_sessions) replaces generate_target_weights. Live fixes - CSI 300 constituents come from China Securities Index's file; the Wikipedia table was 122 names off the real index. - S&P 500 change log moved to "Historical components of the S&P 500". - data_manager: tickers missing from a partial Yahoo batch are retried alone before being marked unavailable (CEG, GEV, SNDK ... were dropped). update() takes fields=[...] and stores high/low/volume panels. - Whole-lot orders go to the nearest lot within cash (CN books held ~62% cash); held names are resized only beyond RESIZE_BAND (10%), so books no longer pay a ticket fee to trade daily drift. Web backtests - backtest_runner.py replaces the duplicated bridge/precompute code: PIT membership in both markets (data/csi300_history.json), stock universe limited to index members (ETFs and benchmarks in the shared panel were pickable), and the book kept like the live trader (simulate_live_book: signal timing, drift, order floor, resize band, fees on orders placed). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -20,6 +20,7 @@ import os
|
||||
import urllib.request
|
||||
from datetime import date, datetime
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
CACHE_DIR = "data"
|
||||
@@ -30,34 +31,51 @@ _HEADERS = {"User-Agent": "Mozilla/5.0 (quant-backtest)"}
|
||||
# Fetch + parse Wikipedia
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _fetch_sp500_tables() -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
"""Return (current_list, changes_log) from the S&P 500 Wikipedia page."""
|
||||
url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
|
||||
SP500_LIST_URL = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
|
||||
# The change log moved off the list page in 2026; the list page is still tried
|
||||
# first in case it moves back.
|
||||
SP500_CHANGES_URLS = [
|
||||
SP500_LIST_URL,
|
||||
"https://en.wikipedia.org/wiki/Historical_components_of_the_S%26P_500",
|
||||
]
|
||||
|
||||
|
||||
def _read_tables(url: str) -> list[pd.DataFrame]:
|
||||
req = urllib.request.Request(url, headers=_HEADERS)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
html = resp.read().decode("utf-8")
|
||||
tables = pd.read_html(io.StringIO(html))
|
||||
current = tables[0]
|
||||
change_tables = [
|
||||
t for t in tables[1:]
|
||||
if any("Added" in "_".join(map(str, c if isinstance(c, tuple) else (c,))) for c in t.columns)
|
||||
]
|
||||
if not change_tables:
|
||||
raise ValueError("S&P 500 change log table not found on the Wikipedia page")
|
||||
changes = change_tables[0]
|
||||
changes.columns = [
|
||||
"_".join(c).strip() if isinstance(c, tuple) else c
|
||||
for c in changes.columns
|
||||
]
|
||||
changes.columns = [
|
||||
c.replace("Effective Date_Effective Date", "Date") for c in changes.columns
|
||||
]
|
||||
return current, changes
|
||||
return pd.read_html(io.StringIO(html))
|
||||
|
||||
|
||||
def _find_changes_table(tables: list[pd.DataFrame]) -> pd.DataFrame | None:
|
||||
for t in tables:
|
||||
if any("Added" in "_".join(map(str, c if isinstance(c, tuple) else (c,))) for c in t.columns):
|
||||
changes = t.copy()
|
||||
changes.columns = [
|
||||
"_".join(c).strip() if isinstance(c, tuple) else c
|
||||
for c in changes.columns
|
||||
]
|
||||
changes.columns = [
|
||||
c.replace("Effective Date_Effective Date", "Date") for c in changes.columns
|
||||
]
|
||||
return changes
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_sp500_tables() -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
"""Return (current_list, changes_log) from Wikipedia."""
|
||||
current = _read_tables(SP500_LIST_URL)[0]
|
||||
for url in SP500_CHANGES_URLS:
|
||||
changes = _find_changes_table(_read_tables(url))
|
||||
if changes is not None:
|
||||
return current, changes
|
||||
raise ValueError("S&P 500 change log table not found on Wikipedia")
|
||||
|
||||
|
||||
def _normalize_ticker(t: str) -> str:
|
||||
"""Yahoo Finance ticker format: BRK.B → BRK-B."""
|
||||
return str(t).replace(".", "-").strip()
|
||||
# Some change-log cells carry a trailing " |" separator.
|
||||
return str(t).split("|")[0].replace(".", "-").strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -243,3 +261,101 @@ def mask_prices(prices: pd.DataFrame,
|
||||
"""
|
||||
mask = membership_mask(prices.index, intervals, tickers=list(prices.columns))
|
||||
return prices.where(mask)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSI 300 (China A-shares)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# data/csi300_history.json holds monthly constituent snapshots
|
||||
# {"snapshots": {"YYYY-MM-DD": [yahoo tickers], ...}}
|
||||
# seeded from baostock's query_hs300_stocks (first trading day of each month,
|
||||
# 2008 onward). A snapshot applies from its date until the next one, so an
|
||||
# index change is picked up within a month. load_csi300_history() appends a
|
||||
# snapshot whenever the current constituent list (universe.get_csi300) differs
|
||||
# from the latest one, which keeps the file current without baostock.
|
||||
|
||||
def _csi300_path() -> str:
|
||||
return os.path.join(CACHE_DIR, "csi300_history.json")
|
||||
|
||||
|
||||
def baostock_to_yahoo(code: str) -> str:
|
||||
"""'sh.600000' -> '600000.SS', 'sz.000001' -> '000001.SZ'."""
|
||||
ex, num = code.split(".")
|
||||
return f"{num}.{'SS' if ex == 'sh' else 'SZ'}"
|
||||
|
||||
|
||||
def load_csi300_history(current: list[str] | None = None) -> dict[str, list[str]]:
|
||||
"""Monthly CSI 300 snapshots; appends ``current`` (or today's list) if it changed."""
|
||||
path = _csi300_path()
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
snaps = data["snapshots"]
|
||||
if current is None:
|
||||
try:
|
||||
from universe import get_csi300
|
||||
current = get_csi300()
|
||||
except Exception as exc: # noqa: BLE001 — offline runs use the stored history
|
||||
print(f"--- WARNING: CSI 300 list unavailable ({exc}); using stored history ---")
|
||||
current = None
|
||||
if current:
|
||||
last = snaps[max(snaps)]
|
||||
today = str(date.today())
|
||||
if sorted(current) != sorted(last) and today > max(snaps):
|
||||
snaps[today] = sorted(current)
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f, indent=0)
|
||||
print(f"--- CSI 300 constituents changed; stored snapshot {today} ---")
|
||||
return snaps
|
||||
|
||||
|
||||
def csi300_membership_mask(dates: pd.DatetimeIndex, tickers: list[str],
|
||||
snapshots: dict[str, list[str]] | None = None) -> pd.DataFrame:
|
||||
"""Boolean mask (dates x tickers): True while the ticker was in the CSI 300."""
|
||||
snaps = snapshots if snapshots is not None else load_csi300_history()
|
||||
keys = sorted(snaps)
|
||||
col = {t: i for i, t in enumerate(tickers)}
|
||||
arr = np.zeros((len(dates), len(tickers)), dtype=bool)
|
||||
for i, k in enumerate(keys):
|
||||
start = pd.Timestamp(k)
|
||||
end = pd.Timestamp(keys[i + 1]) if i + 1 < len(keys) else dates[-1] + pd.Timedelta(days=1)
|
||||
rows = (dates >= start) & (dates < end)
|
||||
cols = [col[t] for t in snaps[k] if t in col]
|
||||
if rows.any() and cols:
|
||||
arr[np.ix_(rows, cols)] = True
|
||||
mask = pd.DataFrame(arr, index=dates, columns=tickers)
|
||||
# Tickers never in the index (benchmarks, ETFs) pass through unmasked.
|
||||
ever = {t for lst in snaps.values() for t in lst}
|
||||
for t in tickers:
|
||||
if t not in ever:
|
||||
mask[t] = True
|
||||
return mask
|
||||
|
||||
|
||||
def all_csi300_tickers_ever(snapshots: dict[str, list[str]] | None = None,
|
||||
since: str | None = None) -> list[str]:
|
||||
snaps = snapshots if snapshots is not None else load_csi300_history()
|
||||
return sorted({t for k, lst in snaps.items() if since is None or k >= since for t in lst})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Market-generic helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def pit_tickers_ever(market: str, since: str | None = None) -> list[str] | None:
|
||||
"""Every ticker that was an index member (since ``since``), or None when the
|
||||
market has no membership history."""
|
||||
if market == "us":
|
||||
return all_tickers_ever()
|
||||
if market == "cn":
|
||||
return all_csi300_tickers_ever(since=since)
|
||||
return None
|
||||
|
||||
|
||||
def mask_prices_pit(prices: pd.DataFrame, market: str) -> pd.DataFrame:
|
||||
"""NaN out (date, ticker) pairs outside point-in-time index membership."""
|
||||
if market == "us":
|
||||
return mask_prices(prices)
|
||||
if market == "cn":
|
||||
return prices.where(csi300_membership_mask(prices.index, list(prices.columns)))
|
||||
return prices
|
||||
|
||||
Reference in New Issue
Block a user