feat: robust-momentum strategies live; fix live universe, lots and UI backtests
Strategies - BarStrategy protocol (close/open/high/low/volume bars; row d uses bars through d-1) and RobustMomentumStrategy: overnight_recovery_top10 (US), lowrisk_momentum_top15 and lowbeta_sharpe_mom_top20 (CN), top-N with rank hysteresis, 10-day rebalance, weights drift between rebalances. - trader: strategy_fields/strategy_bars/bar_session_weights; one signal path (_signals_for_sessions) replaces generate_target_weights. Live fixes - CSI 300 constituents come from China Securities Index's file; the Wikipedia table was 122 names off the real index. - S&P 500 change log moved to "Historical components of the S&P 500". - data_manager: tickers missing from a partial Yahoo batch are retried alone before being marked unavailable (CEG, GEV, SNDK ... were dropped). update() takes fields=[...] and stores high/low/volume panels. - Whole-lot orders go to the nearest lot within cash (CN books held ~62% cash); held names are resized only beyond RESIZE_BAND (10%), so books no longer pay a ticket fee to trade daily drift. Web backtests - backtest_runner.py replaces the duplicated bridge/precompute code: PIT membership in both markets (data/csi300_history.json), stock universe limited to index members (ETFs and benchmarks in the shared panel were pickable), and the book kept like the live trader (simulate_live_book: signal timing, drift, order floor, resize band, fees on orders placed). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
156
data_manager.py
156
data_manager.py
@@ -265,94 +265,148 @@ def _fill_open(close: pd.DataFrame, open_raw: pd.DataFrame | None) -> pd.DataFra
|
||||
return aligned.fillna(close)
|
||||
|
||||
|
||||
def update(market: str, tickers: list[str],
|
||||
with_open: bool = False) -> pd.DataFrame | tuple[pd.DataFrame, pd.DataFrame]:
|
||||
YAHOO_FIELDS = {"close": "Close", "open": "Open", "high": "High", "low": "Low", "volume": "Volume"}
|
||||
|
||||
|
||||
def _backfill(tickers: list[str], start: str, yahoo_fields: list[str],
|
||||
index: pd.DatetimeIndex) -> tuple[dict[str, pd.DataFrame], list[str]]:
|
||||
"""Download new tickers over ``index``; return (panels, tickers still missing).
|
||||
|
||||
yfinance reports a ticker it failed to fetch (rate limit, proxy error) the
|
||||
same way as a ticker it has no data for, so tickers missing from the first
|
||||
batch are downloaded once more on their own before being given up on.
|
||||
"""
|
||||
Refresh the stored panel for ``tickers`` and return it forward-filled.
|
||||
def fetch(batch: list[str]) -> dict[str, pd.DataFrame]:
|
||||
raw = _download(batch, start=start, fields=yahoo_fields)
|
||||
close = raw["Close"].dropna(axis=1, how="all").reindex(index=index).dropna(axis=1, how="all")
|
||||
return {f: raw.get(f, pd.DataFrame()).reindex(index=index, columns=close.columns)
|
||||
for f in yahoo_fields}
|
||||
|
||||
got = fetch(tickers)
|
||||
missing = [t for t in tickers if t not in got["Close"].columns]
|
||||
if missing and len(missing) < len(tickers):
|
||||
retry = fetch(missing)
|
||||
if not retry["Close"].empty:
|
||||
got = {f: pd.concat([got[f], retry[f]], axis=1) for f in yahoo_fields}
|
||||
missing = [t for t in missing if t not in retry["Close"].columns]
|
||||
return got, missing
|
||||
|
||||
|
||||
def update(market: str, tickers: list[str], with_open: bool = False,
|
||||
fields: list[str] | None = None):
|
||||
"""
|
||||
Refresh the stored panels for ``tickers`` and return them forward-filled.
|
||||
|
||||
- First run: downloads 10 years of history.
|
||||
- Every run: re-downloads the last ``REFRESH_DAYS`` calendar days for the
|
||||
requested tickers already in the panel and overwrites them.
|
||||
- New tickers are backfilled over the panel's full date range; tickers
|
||||
Yahoo has no data for are remembered and retried weekly.
|
||||
- A field requested for the first time (e.g. ``high``) is downloaded for
|
||||
every ticker in the stored close panel.
|
||||
|
||||
Returns the close panel, or ``(close, open)`` when ``with_open`` is True.
|
||||
Returns the close panel, ``(close, open)`` when ``with_open`` is True, or
|
||||
``{field: panel}`` when ``fields`` is given. Open/high/low fall back to the
|
||||
close where a bar is missing; volume is returned raw (NaN = no print).
|
||||
"""
|
||||
fields = ["Close", "Open"] if with_open else ["Close"]
|
||||
names = list(fields) if fields is not None else (["close", "open"] if with_open else ["close"])
|
||||
if "close" not in names:
|
||||
names = ["close"] + names
|
||||
unknown = [f for f in names if f not in YAHOO_FIELDS]
|
||||
if unknown:
|
||||
raise ValueError(f"Unsupported market data field(s): {unknown}")
|
||||
yahoo = [YAHOO_FIELDS[f] for f in names]
|
||||
|
||||
with _panel_lock(market):
|
||||
close_raw = load_raw(market, "close")
|
||||
open_raw = load_raw(market, "open") if with_open else None
|
||||
raw = {f: load_raw(market, f) for f in names}
|
||||
unavailable = _load_unavailable(market)
|
||||
today = datetime.now()
|
||||
|
||||
if close_raw is None:
|
||||
if raw["close"] is None:
|
||||
start = (today - timedelta(days=365 * 10)).strftime("%Y-%m-%d")
|
||||
downloaded = _download(tickers, start=start, fields=fields)
|
||||
close_raw = _merge_refresh(None, downloaded["Close"], None)
|
||||
if with_open:
|
||||
open_raw = _merge_refresh(None, downloaded.get("Open", pd.DataFrame()), None)
|
||||
downloaded = _download(tickers, start=start, fields=yahoo)
|
||||
for f, y in zip(names, yahoo):
|
||||
raw[f] = _merge_refresh(None, downloaded.get(y, pd.DataFrame()), None)
|
||||
else:
|
||||
if with_open and open_raw is None:
|
||||
print("--- Downloading Open prices for the full panel (first time) ---")
|
||||
start = close_raw.index[0].strftime("%Y-%m-%d")
|
||||
opens = _download(list(close_raw.columns), start=start, fields=["Open"])
|
||||
open_raw = _merge_refresh(None, opens["Open"], None)
|
||||
close_raw = raw["close"]
|
||||
first = [(f, y) for f, y in zip(names, yahoo) if raw[f] is None]
|
||||
if first:
|
||||
print(f"--- Downloading {[y for _, y in first]} for the full panel (first time) ---")
|
||||
got = _download(list(close_raw.columns), start=close_raw.index[0].strftime("%Y-%m-%d"),
|
||||
fields=[y for _, y in first])
|
||||
for f, y in first:
|
||||
raw[f] = _merge_refresh(None, got[y], None)
|
||||
|
||||
known = [t for t in tickers if t in close_raw.columns]
|
||||
if known:
|
||||
window_start = close_raw.index[-1] - pd.Timedelta(days=REFRESH_DAYS)
|
||||
fresh = _download(known, start=window_start.strftime("%Y-%m-%d"), fields=fields)
|
||||
fresh = _download(known, start=window_start.strftime("%Y-%m-%d"), fields=yahoo)
|
||||
if not fresh["Close"].empty:
|
||||
close_raw = _merge_refresh(close_raw, fresh["Close"], window_start)
|
||||
if with_open:
|
||||
open_raw = _merge_refresh(open_raw, fresh.get("Open", pd.DataFrame()), window_start)
|
||||
for f, y in zip(names, yahoo):
|
||||
raw[f] = _merge_refresh(raw[f], fresh.get(y, pd.DataFrame()), window_start)
|
||||
else:
|
||||
print("--- Refresh download returned no data; keeping stored panel ---")
|
||||
|
||||
cutoff = (today - timedelta(days=UNAVAILABLE_RETRY_DAYS)).strftime("%Y-%m-%d")
|
||||
new_tickers = [t for t in tickers
|
||||
if t not in close_raw.columns and unavailable.get(t, "") < cutoff]
|
||||
if t not in raw["close"].columns and unavailable.get(t, "") < cutoff]
|
||||
if new_tickers:
|
||||
start = close_raw.index[0].strftime("%Y-%m-%d")
|
||||
backfill = _download(new_tickers, start=start, fields=fields)
|
||||
downloaded_any = not backfill["Close"].dropna(how="all").empty
|
||||
got = backfill["Close"].dropna(axis=1, how="all")
|
||||
got = got.reindex(index=close_raw.index)
|
||||
got = got.dropna(axis=1, how="all")
|
||||
for t in new_tickers:
|
||||
if t in got.columns:
|
||||
unavailable.pop(t, None)
|
||||
elif downloaded_any:
|
||||
# Only a ticker missing from an otherwise successful
|
||||
# download is unavailable; a failed request is retried.
|
||||
got, missing = _backfill(new_tickers, raw["close"].index[0].strftime("%Y-%m-%d"),
|
||||
yahoo, raw["close"].index)
|
||||
added = list(got["Close"].columns)
|
||||
for t in added:
|
||||
unavailable.pop(t, None)
|
||||
# A batch that returned nothing at all is a failed request, not
|
||||
# evidence the tickers have no data: retry those next run.
|
||||
if added:
|
||||
for t in missing:
|
||||
unavailable[t] = today.strftime("%Y-%m-%d")
|
||||
if not got.empty:
|
||||
close_raw = pd.concat([close_raw, got], axis=1)
|
||||
if with_open and open_raw is not None:
|
||||
opens = backfill.get("Open", pd.DataFrame())
|
||||
opens = opens.reindex(index=open_raw.index, columns=got.columns)
|
||||
open_raw = pd.concat([open_raw, opens], axis=1)
|
||||
print(f"--- Backfilled {got.shape[1]} new tickers ---")
|
||||
missing = len(new_tickers) - got.shape[1]
|
||||
if added:
|
||||
for f, y in zip(names, yahoo):
|
||||
panel = got[y].reindex(index=raw[f].index)
|
||||
raw[f] = pd.concat([raw[f], panel], axis=1)
|
||||
print(f"--- Backfilled {len(added)} new tickers ---")
|
||||
if missing:
|
||||
print(f"--- {missing} tickers have no Yahoo data; retry after "
|
||||
print(f"--- {len(missing)} tickers have no Yahoo data; retry after "
|
||||
f"{UNAVAILABLE_RETRY_DAYS} days ---")
|
||||
|
||||
close_raw = close_raw.loc[:, ~close_raw.columns.duplicated()]
|
||||
_write_atomic(close_raw, _data_path(market, "close"))
|
||||
print(f"--- Saved {close_raw.shape[0]} days x {close_raw.shape[1]} tickers to "
|
||||
for f in names:
|
||||
raw[f] = raw[f].loc[:, ~raw[f].columns.duplicated()]
|
||||
_write_atomic(raw[f], _data_path(market, f))
|
||||
print(f"--- Saved {raw['close'].shape[0]} days x {raw['close'].shape[1]} tickers to "
|
||||
f"{_data_path(market, 'close')} ---")
|
||||
if with_open and open_raw is not None:
|
||||
open_raw = open_raw.loc[:, ~open_raw.columns.duplicated()]
|
||||
_write_atomic(open_raw, _data_path(market, "open"))
|
||||
_save_unavailable(market, unavailable)
|
||||
|
||||
close = close_raw.ffill()
|
||||
close = raw["close"].ffill()
|
||||
out = {"close": close}
|
||||
for f in names:
|
||||
if f in ("open", "high", "low"):
|
||||
out[f] = _fill_open(close, raw[f])
|
||||
elif f == "volume":
|
||||
out[f] = raw[f].reindex(index=close.index, columns=close.columns)
|
||||
if fields is not None:
|
||||
return out
|
||||
if with_open:
|
||||
return close, _fill_open(close, open_raw)
|
||||
return close, out["open"]
|
||||
return close
|
||||
|
||||
|
||||
def load_fields(market: str, fields: list[str]) -> dict[str, pd.DataFrame]:
|
||||
"""Stored panels without refreshing, aligned and filled the way ``update`` returns them."""
|
||||
close = load(market, "close")
|
||||
out = {"close": close}
|
||||
for f in fields:
|
||||
if f == "close":
|
||||
continue
|
||||
stored = load_raw(market, f)
|
||||
if f == "volume":
|
||||
out[f] = (stored.reindex(index=close.index, columns=close.columns)
|
||||
if stored is not None else pd.DataFrame(np.nan, index=close.index, columns=close.columns))
|
||||
else:
|
||||
out[f] = _fill_open(close, stored)
|
||||
return out
|
||||
|
||||
|
||||
def rebuild(market: str, with_open: bool = True, years: int = 10) -> None:
|
||||
"""Re-download the full history of every ticker in a stored panel.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user