diff --git a/CLAUDE.md b/CLAUDE.md index b364d1c..bbd8934 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,11 +148,11 @@ uv run python -m unittest discover tests **Universe provider** (`universe.py`): Dynamically fetches index constituents (S&P 500 from Wikipedia; CSI 300 from China Securities Index's published constituent file, falling back to the latest `data/csi300_history.json` snapshot, then Wikipedia, whose table lagged the index by ~60 names in 2026) and converts to Yahoo Finance ticker format. Constituent lists are cached daily in `data/universe_*.json`. New markets are added by registering in `UNIVERSES` dict. -**Point-in-time membership** (`universe_history.py`): S&P 500 intervals rebuilt from Wikipedia's "Historical components of the S&P 500" change log into `data/sp500_history.json`; CSI 300 monthly snapshots in `data/csi300_history.json` (seeded from baostock 2008–2026, appended whenever the current list changes). `mask_prices_pit(prices, market)` masks non-member dates. +**Point-in-time membership** (`universe_history.py`): S&P 500 intervals rebuilt from Wikipedia's "Historical components of the S&P 500" change log into `data/sp500_history.json`. The log is sparse before 2011, so current members' earlier additions come from the list page's "Date added"; companies removed before 2011 that the log never recorded stay missing, and the research panel holds only 347 / 367 / 438 / 453 / 484 priced members in mid-2004 / 2006 / 2010 / 2012 / 2016 (about 500 from 2020): US research results before ~2014 are survivor-biased. CSI 300 monthly snapshots in `data/csi300_history.json` (seeded from baostock 2008–2026, appended whenever the current list changes). `mask_prices_pit(prices, market)` masks non-member dates. **Data manager** (`data_manager.py`): Persistent price storage in `data/{market}.csv` (close) and `data/{market}_{open,high,low,volume}.csv`. First run downloads 10 years of history; every run re-downloads the last 30 days. New tickers from index rebalances are backfilled automatically; tickers missing from a partial batch are retried alone before being marked unavailable (retried weekly). `update(market, tickers, with_open=True)` returns (close, open); `update(..., fields=[...])` returns `{field: panel}`. -**Backtest engine** (`main.py`, `backtest_runner.py`): all replay paths use `signals.session_targets` and the cash-funded order ledger in `execution.py`. Price strategies may use the session open and fill at its close; close-only inputs and BarStrategy inputs use prior completed bars. Historical membership restricts ranks and selection separately from price history. Missing quotes and zero volume cannot fill; the book checks available cash after fees, handles whole shares/lots, and applies conservative CN closing-limit restrictions and sell-side stamp tax. Registry replays retain at least 315 prior sessions and score all compared strategies on a common actual window. `--offline` on main/precompute reads a frozen local snapshot. Data coverage and adjusted-share/corporate-action limitations are exposed in API metadata. +**Backtest engine** (`main.py`, `backtest_runner.py`): all replay paths use `signals.session_targets` and the cash-funded order ledger in `execution.py`. Price strategies may use the session open and fill at its close; close-only inputs and BarStrategy inputs use prior completed bars. Historical membership restricts ranks and selection separately from price history. Missing quotes and zero volume cannot fill, except that a held US stock with no print for 5 sessions that has left the index is settled at its last print (`execution.SETTLE_AFTER_SESSIONS`; acquisitions otherwise froze capital forever); the book checks available cash after fees, handles whole shares/lots, and applies conservative CN closing-limit restrictions and sell-side stamp tax. Registry replays retain at least 315 prior sessions and score all compared strategies on a common actual window. `--offline` on main/precompute reads a frozen local snapshot. Data coverage and adjusted-share/corporate-action limitations are exposed in API metadata. `precompute_backtest.py` reruns the ledger for cash-injection scenarios; metrics use unit NAV after actual fees. The API and `web/src/shared/strategyTypes.ts` share this cash-flow contract. `ENGINE_VERSION` in `execution.py` identifies the execution protocol; API cache keys also include its version. diff --git a/backtest_runner.py b/backtest_runner.py index 7021ee1..7f5c184 100644 --- a/backtest_runner.py +++ b/backtest_runner.py @@ -89,6 +89,13 @@ def load_stock_panels(market: str, strategy_names: list[str], years: int | None, "coverage": coverage} +def load_etf_panels(market: str, strategy_names: list[str], *, refresh: bool = True) -> dict: + """Close/open panels in USD for the pure-ETF strategies among ``strategy_names``.""" + request = sorted({t for n in strategy_names for t in ETF_STRATEGY_UNIVERSES[n]} | {"SPY"}) + etfs = _panels("etfs", price_request(market, request), ["close", "open"], refresh) + return {f: convert_quote_currency(p) for f, p in etfs.items()} + + def prepare_strategy(name: str, market: str, panels: dict, cols: list[str], *, member: pd.DataFrame | None = None, years: int | None = None, start=None, end=None, strategy=None) -> dict: @@ -99,6 +106,12 @@ def prepare_strategy(name: str, market: str, panels: dict, cols: list[str], *, if benchmark in close: close = close.loc[close[benchmark].notna()] extra = strategy_extra_tickers(name) + strategy_signal_tickers(name, market) + if name in ETF_STRATEGY_UNIVERSES: + # A pure-ETF strategy handed the stock panel finds none of its tickers + # and would replay as an all-cash book instead of failing. + missing = sorted(set(ETF_STRATEGY_UNIVERSES[name]) - (set(cols) & set(close.columns))) + if missing: + raise ValueError(f"{name} needs its ETF universe; missing {missing}") cols = sorted(set(cols + extra) & set(close.columns)) bars = {f: panels[f].reindex(index=close.index, columns=cols) for f in strategy_fields(name)} @@ -115,7 +128,10 @@ def prepare_strategy(name: str, market: str, panels: dict, cols: list[str], *, "warmup_rows": int((close.index < days[0]).sum()), "resize_band": strategy_resize_band(name), "previous_prices": close.reindex(columns=targets.columns).ffill().shift(1).loc[days], - "volume": panels["volume"].reindex(index=days, columns=targets.columns) if "volume" in panels else None} + "volume": panels["volume"].reindex(index=days, columns=targets.columns) if "volume" in panels else None, + # Tickers outside the membership history (ETFs) count as members: never settled. + "member": (member.reindex(index=days, columns=targets.columns, fill_value=True) + if member is not None else None)} def replay_prepared(plan: dict, capital: float, tx_cost: float, fixed_fee: float, @@ -124,6 +140,7 @@ def replay_prepared(plan: dict, capital: float, tx_cost: float, fixed_fee: float resize_band: float | None = None, cash_returns=None) -> dict: result = simulate_book(plan["targets"], plan["prices"], capital, market=plan["market"], previous_prices=plan.get("previous_prices"), volume=plan.get("volume"), + member=plan.get("member"), transaction_cost=tx_cost, fixed_fee=fixed_fee, fee_base=fee_base, fee_per_share=fee_per_share, integer_shares=integer_shares, @@ -167,11 +184,7 @@ def run_market(market: str, strategy_names: list[str], years: int | None, capita stocks = [n for n in strategy_names if n not in ETF_STRATEGY_UNIVERSES] stock = load_stock_panels(market, stocks, years, refresh=refresh) if stocks else None etf_names = [n for n in strategy_names if n in ETF_STRATEGY_UNIVERSES] - etfs = None - if etf_names: - request = sorted(set(t for n in etf_names for t in ETF_STRATEGY_UNIVERSES[n]) | {"SPY"}) - etfs = _panels("etfs", price_request(market, request), ["close", "open"], refresh) - etfs = {f: convert_quote_currency(p) for f, p in etfs.items()} + etfs = load_etf_panels(market, etf_names, refresh=refresh) if etf_names else None histories = [] for panel, benchmark in ((stock["panels"], stock["benchmark"]) if stock else (None, None), (etfs, "SPY")): @@ -213,7 +226,9 @@ def run_market(market: str, strategy_names: list[str], years: int | None, capita curve.insert(0, {"date": str(first.date()), "value": capital}) coverage = stock["coverage"].loc[first:last].dropna() if stock else pd.Series(dtype=float) warnings = ["使用复权价格:历史整股、整手及公司行动的模拟仍有近似。", - "缺少完整退市结算和市场冲击数据,收益为研究估计。"] + ("美股持仓停止报价 5 个交易日且已移出指数后按最后成交价结算;" if market == "us" + else "停牌或退市的 A 股持仓保持冻结,不做退市结算;") + + "缺少市场冲击数据,收益为研究估计。"] if market == "cn": warnings.append("沪深300历史成分股按月记录;缺少完整历史ST状态及集合竞价数据,涨跌停成交采用保守限制。") if common_start > requested_start: diff --git a/execution.py b/execution.py index 0d3b4f9..ca0495d 100644 --- a/execution.py +++ b/execution.py @@ -7,13 +7,31 @@ import math import numpy as np import pandas as pd -ENGINE_VERSION = "cash-ledger-pit-v2" +ENGINE_VERSION = "cash-ledger-pit-v3" + +# A held US stock that stops printing (cash acquisition, delisting) is sold at +# its last print once it has gone this many sessions without one and is no +# longer an index member; a halted member stays frozen. Without this the +# position stays at its last mark forever and the capital is never +# reinvested. A-shares are excluded: a suspended CSI 300 name can leave the +# index while it still cannot be sold. +SETTLE_AFTER_SESSIONS = 5 def portfolio_value(holdings: dict, prices: dict, cash: float) -> float: return cash + sum(shares * prices.get(t, 0.0) for t, shares in holdings.items()) +def delisting_settlements(market, holdings: dict, sessions_without_print: dict, + members: pd.Series | None) -> set: + """Held names to sell at their last print today. ``members`` is the day's + index membership; tickers outside it (ETFs) are never settled.""" + if market != "us" or members is None: + return set() + return {t for t in holdings + if sessions_without_print.get(t, 0) >= SETTLE_AFTER_SESSIONS and not members.get(t, True)} + + def lot_rule(ticker: str, lot_size: int) -> tuple[int, int]: # STAR orders start at 200 shares, then increase by one (SSE rule 6.7). return (1, 200) if lot_size == 100 and ticker.startswith("688") and ticker.endswith(".SS") else (lot_size, lot_size) @@ -282,8 +300,11 @@ def simulate_book(targets: pd.DataFrame, prices: pd.DataFrame, capital: float, can_buy: pd.DataFrame | None = None, can_sell: pd.DataFrame | None = None, sell_tax: pd.Series | None = None, market: str | None = None, previous_prices: pd.DataFrame | None = None, - volume: pd.DataFrame | None = None, **options): - """Targets are executed at their row's close; stale marks never authorize fills.""" + volume: pd.DataFrame | None = None, + member: pd.DataFrame | None = None, **options): + """Targets are executed at their row's close; stale marks never authorize + fills, except a delisting settlement (``delisting_settlements``, which + needs the point-in-time ``member`` panel).""" raw = prices if fresh_prices is None else fresh_prices.reindex_like(prices) marks = prices.ffill() previous = marks.shift(1) if previous_prices is None else previous_prices.reindex_like(prices) @@ -295,6 +316,7 @@ def simulate_book(targets: pd.DataFrame, prices: pd.DataFrame, capital: float, state = {"holdings": {}, "cash": float(capital), "trade_log": []} equity, units, values, cash = [], [], [], [] unit_count = float(capital) + without_print = {} for i, day in enumerate(prices.index): quote = marks.iloc[i].dropna().to_dict() state["cash"] *= 1 + float(rates.iloc[i]) @@ -307,12 +329,20 @@ def simulate_book(targets: pd.DataFrame, prices: pd.DataFrame, capital: float, fresh = set(raw.columns[(raw.iloc[i].notna() & (raw.iloc[i] > 0)).to_numpy()]) allowed_buy, allowed_sell, tax = market_permissions(market, day, raw.iloc[i], previous.iloc[i], volume.loc[day] if volume is not None else None) + without_print = {t: 0 if t in fresh else without_print.get(t, 0) + 1 for t in state["holdings"]} + settle = delisting_settlements(market, state["holdings"], without_print, + member.loc[day] if member is not None else None) row = targets.iloc[i] - rebalance(state, row[row != 0].to_dict(), quote, fresh, - can_buy=set(can_buy.columns[can_buy.loc[day]]) if can_buy is not None else allowed_buy, - can_sell=set(can_sell.columns[can_sell.loc[day]]) if can_sell is not None else allowed_sell, - sell_tax=float(sell_tax.loc[day]) if sell_tax is not None else tax, - trade_date=str(day.date()), **options) + wanted = {t: w for t, w in row[row != 0].items() if t not in settle} + log = rebalance(state, wanted, quote, fresh | settle, + can_buy=set(can_buy.columns[can_buy.loc[day]]) if can_buy is not None else allowed_buy, + can_sell=(set(can_sell.columns[can_sell.loc[day]]) if can_sell is not None + else allowed_sell) | settle, + sell_tax=float(sell_tax.loc[day]) if sell_tax is not None else tax, + trade_date=str(day.date()), **options) + for trade in log: + if trade["ticker"] in settle: + trade["settled"] = True value = portfolio_value(state["holdings"], quote, state["cash"]) equity.append(value) units.append(value / unit_count * capital) diff --git a/main.py b/main.py index 7355a63..be1d910 100644 --- a/main.py +++ b/main.py @@ -265,6 +265,7 @@ def main() -> None: run = simulate_book(targets.loc[days], strat_data.loc[days], initial_capital, previous_prices=strat_data.ffill().shift(1).loc[days], volume=panels["volume"].reindex(index=days, columns=strat_data.columns), + member=member.reindex(index=days, columns=strat_data.columns, fill_value=True), market=args.market, integer_shares=True, lot_size=100 if args.market == "cn" else 1, fixed_fee=args.fixed_fee, min_trade_value=50, resize_band=.10) diff --git a/research/lab/factors.py b/research/lab/factors.py index 2f10ab3..6cd0d96 100644 --- a/research/lab/factors.py +++ b/research/lab/factors.py @@ -445,9 +445,12 @@ def trend_hzz(p): b = np.linalg.lstsq(Xo, y[ok].values, rcond=None)[0][1:] betas.append(pd.Series(b, index=lags, name=d)) B = pd.DataFrame(betas) - # the regression at month d uses returns through d+21, so it is usable at d+21; - # shift by one month, then average the last 12 known betas - Ebeta = B.shift(1).rolling(12, min_periods=6).mean() + # the regression at month-end d uses returns through d+21 sessions, so its + # betas are known from that session on (a month can be shorter than 21 + # sessions); average the last 12 known betas + known = c.index.get_indexer(B.index) + 21 + B = B[known < len(c.index)].set_axis(c.index[known[known < len(c.index)]], axis=0) + Ebeta = B.rolling(12, min_periods=6).mean() Ed = Ebeta.reindex(c.index, method="ffill") return sum(feats[L].mul(Ed[L], axis=0) for L in lags) diff --git a/research/lab/panel.py b/research/lab/panel.py index ead25ac..ab17306 100644 --- a/research/lab/panel.py +++ b/research/lab/panel.py @@ -10,6 +10,7 @@ Panels are cached in data/research/*.panel.pkl; delete them after refetching. from __future__ import annotations +import hashlib import json import os import pickle @@ -103,11 +104,18 @@ def _tiingo_sig() -> list: return [len(files), round(mtime, 3)] +def _membership_sig() -> str: + with open(os.path.join("data", "sp500_history.json")) as fh: + intervals = json.load(fh)["intervals"] + return hashlib.sha256(json.dumps(intervals, sort_keys=True).encode()).hexdigest()[:16] + + def load_us() -> Panel: - """Cached US panel, rebuilt whenever the Tiingo overlay changes.""" + """Cached US panel, rebuilt whenever the Tiingo overlay or the S&P 500 + membership intervals change.""" path = os.path.join(RAW_DIR, "us.panel.pkl") key = os.path.join(RAW_DIR, "us.panel.key") - sig = _tiingo_sig() + sig = _tiingo_sig() + [_membership_sig()] if os.path.exists(path) and os.path.exists(key): with open(key) as fh: if json.load(fh) == sig: diff --git a/research/lab/registered.py b/research/lab/registered.py index 5c293c9..93f5813 100644 --- a/research/lab/registered.py +++ b/research/lab/registered.py @@ -11,8 +11,9 @@ import pandas as pd import metrics import data_manager -from backtest_runner import (ENGINE_VERSION, load_stock_panels, prepare_strategy, replay_prepared) -from trader import make_strategy +from backtest_runner import (ENGINE_VERSION, load_etf_panels, load_stock_panels, prepare_strategy, + replay_prepared) +from trader import ETF_STRATEGY_UNIVERSES, make_strategy from strategies.robust_momentum import RobustMomentumStrategy from research.lab.trials import append_trials @@ -26,8 +27,19 @@ def evaluate_registered(args): market = 'cn' if args.market.startswith('cn') else 'us' destination = Path(args.output or f'data/research/results/registered_{market}') destination.mkdir(parents=True, exist_ok=True) - panels = load_stock_panels(market, args.registered, None, refresh=False) - end = args.end or str(panels['panels']['close'].index[-1].date()) + etf_names = [n for n in args.registered if n in ETF_STRATEGY_UNIVERSES] + stock_names = [n for n in args.registered if n not in ETF_STRATEGY_UNIVERSES] + stock = load_stock_panels(market, stock_names, None, refresh=False) if stock_names else None + etfs = load_etf_panels(market, etf_names, refresh=False) if etf_names else None + + def source(name): + """(panels, universe, membership) the strategy is replayed on, as in run_market.""" + if name in ETF_STRATEGY_UNIVERSES: + return etfs, ETF_STRATEGY_UNIVERSES[name], None + return stock['panels'], stock['stocks'], stock['member'] + + ends = [p['close'].index[-1] for p in ([stock['panels']] if stock else []) + ([etfs] if etfs else [])] + end = args.end or str(min(ends).date()) start = args.start or '2018-01-01' profiles = ['baseline', 'resize20', 'buffer3'] + (['cash_shy'] if market == 'us' else []) protocol = { @@ -40,8 +52,10 @@ def evaluate_registered(args): for p in [Path('execution.py'), Path('signals.py'), Path('backtest_runner.py')] + sorted(Path('strategies').glob('*.py'))}, 'data_hashes': {str(p): hashlib.sha256(p.read_bytes()).hexdigest() - for p in [Path('data') / f'{market}.csv', Path('data') / f'{market}_open.csv', - Path('data/sp500_history.json' if market == 'us' else 'data/csi300_history.json')]}, + for p in ([Path('data') / f'{market}.csv', Path('data') / f'{market}_open.csv', + Path('data/sp500_history.json' if market == 'us' else 'data/csi300_history.json')] + if stock_names else []) + + ([Path('data/etfs.csv'), Path('data/etfs_open.csv')] if etf_names else [])}, } protocol_path = destination / 'protocol.json' if protocol_path.exists(): @@ -52,7 +66,8 @@ def evaluate_registered(args): 'fee_base': 1.88 if market == 'us' else 0, 'fee_per_share': .009 if market == 'us' else 0} for name in args.registered: base_strategy = make_strategy(name, market) - base_plan = prepare_strategy(name, market, panels['panels'], panels['stocks'], member=panels['member'], + panels, universe, member = source(name) + base_plan = prepare_strategy(name, market, panels, universe, member=member, start=start, end=end, strategy=base_strategy) if base_plan['warmup_rows'] < 315: raise ValueError('Registered comparison requires >=315 pre-window sessions; move --start or add history') @@ -66,17 +81,17 @@ def evaluate_registered(args): if not isinstance(base_strategy, RobustMomentumStrategy): continue changed = copy.deepcopy(base_strategy); changed.buffer = 3.0 - plan = prepare_strategy(name, market, panels['panels'], panels['stocks'], member=panels['member'], + plan = prepare_strategy(name, market, panels, universe, member=member, start=start, end=end, strategy=changed) elif profile == 'cash_shy': - if 'SHY' not in panels['panels']['close']: + if 'SHY' not in panels['close']: raise ValueError('SHY bars are required for a traded cash sleeve') plan = dict(base_plan) plan['prices'] = base_plan['prices'].copy() - plan['prices']['SHY'] = panels['panels']['close']['SHY'].reindex(dates) + plan['prices']['SHY'] = panels['close']['SHY'].reindex(dates) if base_plan.get('volume') is not None: plan['volume'] = base_plan['volume'].copy() - plan['volume']['SHY'] = panels['panels']['volume']['SHY'].reindex(dates) + plan['volume']['SHY'] = panels['volume']['SHY'].reindex(dates) target = base_plan['targets'].reindex(columns=plan['prices'].columns, fill_value=0).copy() target['SHY'] += (1-target.sum(axis=1)).clip(lower=0) plan['targets'] = target diff --git a/tests/test_causality_contract.py b/tests/test_causality_contract.py new file mode 100644 index 0000000..47b057c --- /dev/null +++ b/tests/test_causality_contract.py @@ -0,0 +1,86 @@ +"""Every registered strategy is causal on the production signal path. + +The targets a replay or the live trader trades on a session must not change +when later bars are removed. ``trader._signals_for_sessions`` and the batch +replay compute all sessions in one pass, which equals running each session on +its own only for causal strategies, so a new strategy that reads a later bar +(a missing shift, a full-sample statistic, a backward fill) fails here. +""" + +import unittest + +import numpy as np +import pandas as pd + +import trader +from backtest_runner import prepare_strategy +from universe import UNIVERSES + +N_DAYS, N_STOCKS = 700, 60 +CUTS = (560, 640) # truncation points (row positions) +START = 480 # first compared session + + +def _panel(seed: int = 7) -> tuple[dict[str, pd.DataFrame], pd.DataFrame, list[str]]: + rng = np.random.default_rng(seed) + stocks = [f"S{i:02d}" for i in range(N_STOCKS)] + etfs = sorted({t for u in trader.ETF_STRATEGY_UNIVERSES.values() for t in u} + | {t for u in trader.MIXED_STRATEGY_EXTRA_TICKERS.values() for t in u} + | {UNIVERSES[m]["benchmark"] for m in UNIVERSES}) + cols = stocks + etfs + idx = pd.bdate_range("2023-01-02", periods=N_DAYS) + r = rng.normal(0.0004, 0.02, (N_DAYS, len(cols))) + close = pd.DataFrame(100 * np.exp(np.cumsum(r, axis=0)), index=idx, columns=cols) + opens = close.shift(1).bfill() * np.exp(rng.normal(0, 0.005, close.shape)) + high = np.maximum(opens, close) * (1 + rng.uniform(0, 0.02, close.shape)) + low = np.minimum(opens, close) * (1 - rng.uniform(0, 0.02, close.shape)) + volume = pd.DataFrame(rng.uniform(1e5, 1e6, close.shape), index=idx, columns=cols) + # Index changes: five names join late (price history before admission is + # observable), five leave, one stops printing like an acquired stock. + member = pd.DataFrame(True, index=idx, columns=stocks) + member.iloc[:300, :5] = False + member.iloc[500:, 5:10] = False + for frame in (close, opens, high, low): + frame.iloc[600:, 10] = np.nan + volume.iloc[600:, 10] = np.nan + panels = {"close": close, "open": opens, "high": high, "low": low, "volume": volume} + return panels, member, stocks + + +class CausalityContract(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.panels, cls.member, cls.stocks = _panel() + + def _targets(self, name: str, market: str, end) -> pd.DataFrame: + pure_etf = trader.canonical_strategy_name(name) in trader.ETF_STRATEGY_UNIVERSES + cols = trader.ETF_STRATEGY_UNIVERSES[name] if pure_etf else self.stocks + start = self.panels["close"].index[START] + return prepare_strategy(name, market, self.panels, list(cols), + member=None if pure_etf else self.member, + start=start, end=end)["targets"] + + def test_targets_ignore_later_bars(self): + idx = self.panels["close"].index + leaks, idle = [], [] + for market in ("us", "cn"): + for name in trader.STRATEGY_REGISTRY: + if name in trader.AGENT_REVIEW_STRATEGIES or not trader.strategy_supports_market(name, market): + continue + full = self._targets(name, market, None) + if not (full.to_numpy() > 0).any(): + idle.append(f"{market}:{name}") + for cut in CUTS: + part = self._targets(name, market, idx[cut]) + cols = full.columns.union(part.columns) + a = full.reindex(index=part.index, columns=cols).fillna(0.0) + b = part.reindex(columns=cols).fillna(0.0) + worst = float((a - b).abs().to_numpy().max()) + if worst > 1e-9: + leaks.append(f"{market}:{name} (cut {idx[cut].date()}, max diff {worst:.3g})") + self.assertEqual(leaks, [], "targets depend on bars after their session") + self.assertEqual(idle, [], "never invested on the synthetic panel, so the check is vacuous") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_execution_contract.py b/tests/test_execution_contract.py index 1984d59..518c2be 100644 --- a/tests/test_execution_contract.py +++ b/tests/test_execution_contract.py @@ -52,6 +52,31 @@ class ExecutionContracts(unittest.TestCase): self.assertAlmostEqual(state["daily_equity"][str(day.date())], batch["equity"].loc[day], places=2) self.assertEqual(state["trade_log"], batch["trade_log"]) + def test_a_delisted_stock_is_settled_at_its_last_print_in_both_ledgers(self): + dates = pd.bdate_range("2026-01-05", periods=9) + # A is acquired: last print on day 2, out of the index from day 4. + # H halts on the same day but stays a member, so it stays frozen. + prices = pd.DataFrame({"A": [100, 101, 102] + [np.nan] * 6, + "H": [50, 50, 51] + [np.nan] * 6, + "B": [20.] * 9}, index=dates) + member = pd.DataFrame(True, index=dates, columns=prices.columns) + member.loc[dates[4]:, "A"] = False + targets = pd.DataFrame([[.4, .3, .3]] * 9, index=dates, columns=prices.columns) + options = dict(transaction_cost=.001, fixed_fee=2., integer_shares=True, resize_band=.1) + batch = simulate_book(targets, prices, 10000, market="us", member=member, **options) + settled = [(t["ticker"], t["date"], t["price"]) for t in batch["trade_log"] if t.get("settled")] + self.assertEqual(settled, [("A", str(dates[7].date()), 102.0)]) + self.assertIn("H", batch["holdings"]) + state = trader.init_state("us", "test", 10000) + fees = {"tx_cost": .001, "fixed_fee": 2.} + for day in dates: + trader._run_session(state, "us", day, "B", prices.ffill(), prices, fees, True, targets.loc[day], + member_row=member.loc[day]) + self.assertAlmostEqual(state["daily_equity"][str(day.date())], batch["equity"].loc[day], places=2) + self.assertEqual(state["trade_log"], batch["trade_log"]) + china = simulate_book(targets, prices, 10000, market="cn", member=member, **options) + self.assertFalse(any(t.get("settled") for t in china["trade_log"])) + def test_contributions_are_executed_and_fees_affect_unit_returns(self): dates = pd.bdate_range("2026-01-05", periods=3) prices = pd.DataFrame({"A": [100., 100., 100.]}, index=dates) diff --git a/trader.py b/trader.py index 40d0e69..3456480 100644 --- a/trader.py +++ b/trader.py @@ -477,6 +477,14 @@ def index_members(market: str, days: pd.DatetimeIndex) -> pd.DataFrame | None: return universe_history.pit_membership(market, days) +def settlement_members(market: str, strategy_name: str, days: pd.DatetimeIndex) -> pd.DataFrame | None: + """Index membership on ``days`` for delisting settlement; None where it does + not apply (A-shares, pure-ETF strategies).""" + if market != "us" or strategy_data_market(market, strategy_name) == "etfs": + return None + return index_members(market, days) + + def pit_candidates(market: str, strategy_name: str, since: pd.Timestamp | None) -> list[str]: """Tickers that were index members on any day from ``since`` to today, so a catch-up or replay has prices for names that have left the index since.""" @@ -857,13 +865,16 @@ def _signals_for_sessions(strategy_name: str, open_data: pd.DataFrame, def _run_session(state: dict, market: str, day: pd.Timestamp, benchmark: str, close_data: pd.DataFrame, close_raw: pd.DataFrame, fees: dict, integer_shares: bool, signal_row: pd.Series, - volume_row: pd.Series | None = None) -> dict: + volume_row: pd.Series | None = None, + member_row: pd.Series | None = None) -> dict: """Execute ``signal_row`` (targets from ``day``'s open; a BarStrategy: bars through the previous session, see ``session_signals``) at ``day``'s close and record the day. Tickers without a real closing print on ``day`` (forward-filled price) are - valued but not traded. + valued but not traded, except a held US stock that stopped printing and + left the index (``member_row``), which is settled at its last print as in + ``execution.simulate_book``. """ day_str = str(day.date()) if state.get("engine_version") != ENGINE_VERSION: @@ -879,18 +890,30 @@ def _run_session(state: dict, market: str, day: pd.Timestamp, benchmark: str, ) raw_row = close_raw.loc[day] frozen = {t for t in all_held if t not in raw_row.index or pd.isna(raw_row[t])} + # Imported here so a running monitor's hot reload of this module does not + # need an execution module newer than the one it started with. + from execution import delisting_settlements + upto = close_raw.loc[:day] + without_print = {t: int((upto.index > upto[t].last_valid_index()).sum()) + if t in upto.columns and upto[t].notna().any() else len(upto) + for t in state["holdings"]} + settle = delisting_settlements(market, state["holdings"], without_print, member_row) + target_weights = {t: w for t, w in target_weights.items() if t not in settle} pre_value = portfolio_value(state["holdings"], close_prices, state["cash"]) previous_rows = close_data.loc[close_data.index < day] previous = previous_rows.iloc[-1] if len(previous_rows) else pd.Series(dtype=float) allowed_buy, allowed_sell, sell_tax = market_permissions(market, day, raw_row, previous, volume_row) n_log = len(state["trade_log"]) - actual = rebalance(state, target_weights, close_prices, set(raw_row.dropna().index), + actual = rebalance(state, target_weights, close_prices, set(raw_row.dropna().index) | settle, transaction_cost=fees["tx_cost"], fixed_fee=fees["fixed_fee"], fee_base=fees.get("fee_base", 0), fee_per_share=fees.get("fee_per_share", 0), integer_shares=integer_shares, lot_size=lot_size, trade_date=day_str, resize_band=strategy_resize_band(state["strategy"]), - can_buy=allowed_buy, can_sell=allowed_sell, sell_tax=sell_tax) + can_buy=allowed_buy, can_sell=allowed_sell | settle, sell_tax=sell_tax) + for trade in actual: + if trade["ticker"] in settle: + trade["settled"] = True trades = [{**t, "direction": t["action"], "est_value": t["value"], "shares_delta": t["shares"] * (1 if t["action"] == "BUY" else -1)} for t in actual] post_value = portfolio_value(state["holdings"], close_prices, state["cash"]) @@ -1071,10 +1094,12 @@ def cmd_evening(args): lot_size = MARKET_LOT_SIZE.get(market, 1) first_log = len(state["trade_log"]) volume = data_manager.load_raw(strategy_data_market(market, strategy_name), "volume") + members = settlement_members(market, strategy_name, pd.DatetimeIndex([target_date])) _run_session(state, market, target_date, benchmark, close_data, close_raw, _strategy_fees(args, market), args.integer_shares, pd.Series(target_weights, dtype=float), - volume_row=volume.reindex(close_data.index).loc[target_date] if volume is not None else None) + volume_row=volume.reindex(close_data.index).loc[target_date] if volume is not None else None, + member_row=members.loc[target_date] if members is not None else None) exec_trades = [{**t, "direction": t["action"], "est_value": t["value"]} for t in state["trade_log"][first_log:]] @@ -1463,13 +1488,15 @@ def cmd_simulate(args): total_commission = 0.0 volume = data_manager.load_raw(strategy_data_market(market, strategy_name), "volume") + members = settlement_members(market, strategy_name, trading_days) for i, day in enumerate(trading_days): day_str = str(day.date()) stats = _run_session(state, market, day, benchmark, close_data, close_raw, _strategy_fees(args, market), args.integer_shares, full_weights.loc[day], - volume_row=(volume.loc[day] if day in volume.index else pd.Series(dtype=float)) if volume is not None else None) + volume_row=(volume.loc[day] if day in volume.index else pd.Series(dtype=float)) if volume is not None else None, + member_row=members.loc[day] if members is not None else None) post_value = stats["value"] day_trades, day_commission = stats["trades"], stats["commission"] total_trades += day_trades @@ -1864,10 +1891,12 @@ def _process_sessions(args, state: dict | None, start: pd.Timestamp | None = Non fees = _strategy_fees(args, market) volume = data_manager.load_raw(strategy_data_market(market, strategy_name), "volume") signals = session_signals(market, strategy_name, tickers, todo, close_raw, open_data) + members = settlement_members(market, strategy_name, todo) for day in todo: r = _run_session(state, market, day, benchmark, close_data, close_raw, fees, args.integer_shares, signal_row=signals.loc[day], - volume_row=(volume.loc[day] if day in volume.index else pd.Series(dtype=float)) if volume is not None else None) + volume_row=(volume.loc[day] if day in volume.index else pd.Series(dtype=float)) if volume is not None else None, + member_row=members.loc[day] if members is not None else None) frozen = f" | no fresh price: {', '.join(r['frozen'])}" if r["frozen"] else "" print(f"[{label}] {r['date']} {strategy_name:<34} value {r['value']:>12,.2f} " f"({r['daily_return']:+.2%}) trades {r['trades']:>2} " diff --git a/universe_history.py b/universe_history.py index 88f414b..4ecdc3d 100644 --- a/universe_history.py +++ b/universe_history.py @@ -158,6 +158,8 @@ def build_sp500_history() -> dict[str, list[list[str | None]]]: for t, end in open_end.items(): intervals.setdefault(t, []).append([None, end]) + _close_unlogged_additions(intervals, current) + # Sort intervals per ticker oldest→newest for t, ivs in intervals.items(): ivs.sort(key=lambda iv: (iv[0] or "0000-00-00")) @@ -165,6 +167,30 @@ def build_sp500_history() -> dict[str, list[list[str | None]]]: return intervals +# The change log is sparse before 2011 (0-2 logged additions a year in +# 2001-2006, 6-12 in 2007-2010, against 20-30 real changes), so a current +# member whose addition it missed would count as a member since before the +# log: 80 of the 94 current members added 2001-2010 did, e.g. ISRG, MA, BKNG. +# The list page's "Date added" starts those intervals. Later dates are left to +# the log, which is complete there, because "Date added" also records renames +# and re-listings (META 2013, CPAY 2018) as additions. +UNLOGGED_ADDITIONS_BEFORE = "2011-01-01" + + +def _close_unlogged_additions(intervals: dict, current: pd.DataFrame) -> None: + col = next((c for c in current.columns if "added" in str(c).lower()), None) + if col is None: + print("--- WARNING: no 'Date added' column; pre-2011 additions stay open-start ---") + return + added = pd.to_datetime(current[col], errors="coerce") + for symbol, day in zip(current["Symbol"], added): + if pd.isna(day) or day >= pd.Timestamp(UNLOGGED_ADDITIONS_BEFORE): + continue + for iv in intervals.get(_normalize_ticker(symbol), []): + if iv[0] is None and iv[1] is None: + iv[0] = day.strftime("%Y-%m-%d") + + # --------------------------------------------------------------------------- # Cache I/O # --------------------------------------------------------------------------- diff --git a/web/src/server/strategyService.ts b/web/src/server/strategyService.ts index 3f7fe88..29808d7 100644 --- a/web/src/server/strategyService.ts +++ b/web/src/server/strategyService.ts @@ -17,7 +17,7 @@ function ensureCacheDir(): void { } function configHash(config: BacktestConfig): string { - const serialized = stableStringify({ engine: "cash-ledger-pit-v2", config }); + const serialized = stableStringify({ engine: "cash-ledger-pit-v3", config }); return createHash("md5").update(serialized).digest("hex").slice(0, 12); }