fix: unify backtest execution and add forward validation candidates

Share point-in-time signals and a cash-funded order ledger across backtests and trading. Correct eligibility, missing-price fills, quality factors, regime inputs, warmup windows, and cash-injection accounting. Add reproducible registered-strategy comparisons and paired forward candidates with a 20% resize band.

Validation: 125 Python tests, 23 web tests, TypeScript/Vite build, 87 strategy causality checks, and real-data target/order/equity parity for 10 strategies.
This commit is contained in:
2026-10-01 02:24:01 -07:00
parent 761725d2c8
commit 722cade974
45 changed files with 1561 additions and 1021 deletions

View File

@@ -20,6 +20,7 @@ from research.lab.engine import Costs, simulate
from research.lab.factor_scan import RESULTS, load
from research.lab.report import deflated_sharpe, stats, yearly
from research.lab.stepwise import SUBPERIODS
from research.lab.trials import count_trials
warnings.filterwarnings("ignore")
@@ -81,7 +82,15 @@ def main() -> None:
ap.add_argument("--market", default="us")
ap.add_argument("--add", nargs="*", default=[], help="extra 'name=combo|top|every|buffer'")
ap.add_argument("--n-trials", type=int, default=0, help="override trial count for DSR")
ap.add_argument("--registered", nargs="+", help="Validate registered strategies using the production ledger")
ap.add_argument("--start")
ap.add_argument("--end")
ap.add_argument("--output")
args = ap.parse_args()
if args.registered:
from research.lab.registered import evaluate_registered
evaluate_registered(args)
return
mk = "us" if args.market == "us" else "cn"
p, _ = load(args.market)
cands = dict(CANDIDATES[mk])
@@ -89,8 +98,9 @@ def main() -> None:
name, rest = spec.split("=", 1)
combo, top, every, buf = rest.split("|")
cands[name] = (combo, dict(top_n=int(top), every=int(every), buffer=float(buf)))
trials_path = os.path.join(RESULTS, f"trials_{args.market}.csv")
n_trials = args.n_trials or (len(pd.read_csv(trials_path)) if os.path.exists(trials_path) else 100)
n_trials = args.n_trials or count_trials(RESULTS, args.market)
if n_trials < 1:
raise ValueError("DSR needs a trial ledger or an explicit --n-trials count")
full = (SUBPERIODS[mk][0][0], WINDOW[1])
out = {}
fee, cap = LIVE_FEES[mk]
@@ -115,7 +125,8 @@ def main() -> None:
expo = (0.5 + 0.25 * slow + 0.25 * fast).where(slow + fast < 2, 1.0)
res["window_trend_overlay"] = summarize(phase_runs(p, s, *WINDOW, exposure=expo, **kw), *WINDOW)
dsr = deflated_sharpe(res["full"]["_rets"], n_trials)
res["dsr_full"] = dsr
res["dsr_full"] = {**dsr, "observed_trials_lower_bound": n_trials,
"evidence_role": "development, not independent OOS"}
out[name] = res
w, f = res["window"], res["full"]
print(f"{name:42s} 2016-26: CAGR {w['cagr']*100:5.1f}% [{w['cagr_phase_min']*100:4.1f},{w['cagr_phase_max']*100:4.1f}] "

111
research/lab/registered.py Normal file
View File

@@ -0,0 +1,111 @@
"""Fixed-budget validation of registered strategies on the shared execution engine."""
from __future__ import annotations
import copy
import hashlib
import json
from datetime import datetime, timezone
from pathlib import Path
import pandas as pd
import metrics
import data_manager
from backtest_runner import (ENGINE_VERSION, load_stock_panels, prepare_strategy, replay_prepared)
from trader import make_strategy
from strategies.robust_momentum import RobustMomentumStrategy
from research.lab.trials import append_trials
def _stats(equity):
return {k: v for k, v in metrics.raw_summary(equity).items()
if k in ('annualizedReturn', 'maxDrawdown', 'sharpeRatio', 'annualizedVolatility')}
def evaluate_registered(args):
market = 'cn' if args.market.startswith('cn') else 'us'
destination = Path(args.output or f'data/research/results/registered_{market}')
destination.mkdir(parents=True, exist_ok=True)
panels = load_stock_panels(market, args.registered, None, refresh=False)
end = args.end or str(panels['panels']['close'].index[-1].date())
start = args.start or '2018-01-01'
profiles = ['baseline', 'resize20', 'buffer3'] + (['cash_shy'] if market == 'us' else [])
protocol = {
'created_at': datetime.now(timezone.utc).isoformat(), 'engine': ENGINE_VERSION,
'market': market, 'strategies': args.registered, 'start': start, 'end': end,
'profiles': profiles, 'one_way_costs': [.001, .0025], 'capital': 10000 if market == 'us' else 70000,
'selection_rule': 'CAGR improves at both costs; drawdown deterioration <= 2pp; improves in >=2/3 time blocks',
'role': 'historical development comparison; not independent OOS; do not auto-promote',
'source_hashes': {str(p): hashlib.sha256(p.read_bytes()).hexdigest()
for p in [Path('execution.py'), Path('signals.py'), Path('backtest_runner.py')]
+ sorted(Path('strategies').glob('*.py'))},
'data_hashes': {str(p): hashlib.sha256(p.read_bytes()).hexdigest()
for p in [Path('data') / f'{market}.csv', Path('data') / f'{market}_open.csv',
Path('data/sp500_history.json' if market == 'us' else 'data/csi300_history.json')]},
}
protocol_path = destination / 'protocol.json'
if protocol_path.exists():
raise FileExistsError(f'Choose a new --output to preserve the completed study: {destination}')
protocol_path.write_text(json.dumps(protocol, indent=2))
rows, verdicts = [], []
fees = {'tx_cost': .001, 'fixed_fee': 2 if market == 'us' else 5,
'fee_base': 1.88 if market == 'us' else 0, 'fee_per_share': .009 if market == 'us' else 0}
for name in args.registered:
base_strategy = make_strategy(name, market)
base_plan = prepare_strategy(name, market, panels['panels'], panels['stocks'], member=panels['member'],
start=start, end=end, strategy=base_strategy)
if base_plan['warmup_rows'] < 315:
raise ValueError('Registered comparison requires >=315 pre-window sessions; move --start or add history')
dates = base_plan['prices'].index
cuts = [0, len(dates)//3, 2*len(dates)//3, len(dates)]
for profile in profiles:
plan, band = base_plan, .1
if profile == 'resize20':
band = .2
elif profile == 'buffer3':
if not isinstance(base_strategy, RobustMomentumStrategy):
continue
changed = copy.deepcopy(base_strategy); changed.buffer = 3.0
plan = prepare_strategy(name, market, panels['panels'], panels['stocks'], member=panels['member'],
start=start, end=end, strategy=changed)
elif profile == 'cash_shy':
if 'SHY' not in panels['panels']['close']:
raise ValueError('SHY bars are required for a traded cash sleeve')
plan = dict(base_plan)
plan['prices'] = base_plan['prices'].copy()
plan['prices']['SHY'] = panels['panels']['close']['SHY'].reindex(dates)
if base_plan.get('volume') is not None:
plan['volume'] = base_plan['volume'].copy()
plan['volume']['SHY'] = panels['panels']['volume']['SHY'].reindex(dates)
target = base_plan['targets'].reindex(columns=plan['prices'].columns, fill_value=0).copy()
target['SHY'] += (1-target.sum(axis=1)).clip(lower=0)
plan['targets'] = target
for cost in protocol['one_way_costs']:
run = replay_prepared(plan, protocol['capital'], **{**fees, 'tx_cost': cost}, resize_band=band)
row = {'strategy': name, 'profile': profile, 'cost': cost, **_stats(run['equity']),
'orders': len(run['trade_log']), 'fees': sum(t['commission'] for t in run['trade_log']),
'minimum_cash': float(run['cash'].min()), 'blocks': []}
for a, b in zip(cuts[:-1], cuts[1:]):
row['blocks'].append(_stats(run['equity'].iloc[a:b+1]))
rows.append(row)
append_trials(destination/'trials.jsonl', [row], protocol)
print(name, profile, cost, f"CAGR {row['annualizedReturn']:.2%}",
f"DD {row['maxDrawdown']:.2%}", flush=True)
baseline = {r['cost']: r for r in rows if r['strategy'] == name and r['profile'] == 'baseline'}
for profile in profiles[1:]:
candidates = [r for r in rows if r['strategy'] == name and r['profile'] == profile]
if not candidates:
continue
passed = all(r['annualizedReturn'] > baseline[r['cost']]['annualizedReturn']
and r['maxDrawdown'] >= baseline[r['cost']]['maxDrawdown']-.02
and sum(a['annualizedReturn'] > b['annualizedReturn']
for a,b in zip(r['blocks'],baseline[r['cost']]['blocks'])) >= 2
for r in candidates)
verdicts.append({'strategy': name, 'profile': profile,
'verdict': 'candidate_for_forward_only' if passed else 'retain_baseline'})
summary = {'protocol': protocol, 'comparisons': rows, 'decisions': verdicts,
'boundary': 'Missing historical constituents and adjusted-share execution remain limitations.'}
(destination/'results.json').write_text(json.dumps(summary, indent=2, allow_nan=False))
pd.DataFrame([{k:v for k,v in r.items() if k!='blocks'} for r in rows]).to_csv(destination/'results.csv',index=False)
print('Saved',destination/'results.json',flush=True)
return summary

View File

@@ -1,11 +1,12 @@
"""
Forward stepwise selection of equal-weight rank composites, scored on the
DESIGN period only; the holdout is evaluated once for the final picks.
DESIGN period only; historical validation is reported separately. Previously inspected history
is not a fresh out-of-sample test.
PYTHONPATH=. uv run python -m research.lab.stepwise --market us --max-k 4
Every evaluated composite is appended to the trial ledger
(data/research/results/trials_{market}.csv) so deflated Sharpe can count them.
(data/research/results/trials_{market}.jsonl) so deflated Sharpe can count them.
"""
import argparse
@@ -18,6 +19,7 @@ import pandas as pd
from research.lab.factor_scan import RESULTS, load, phase_avg
from research.lab.factors import FACTORS, zrank
from research.lab.trials import append_trials
warnings.filterwarnings("ignore")
@@ -55,7 +57,8 @@ def main() -> None:
print(f"{len(names)} factor ranks in {time.time() - t:.0f}s", flush=True)
a, b = periods["design"]
ledger = []
subs = SUBPERIODS[mk]
subs = [(max(sa, a), min(sb, b)) for sa, sb in SUBPERIODS[mk]
if max(sa, a) <= min(sb, b)]
def evaluate(combo: tuple[str, ...]) -> dict:
s = sum(ranks[n] for n in combo) / len(combo)
@@ -93,23 +96,25 @@ def main() -> None:
best_overall += ranked[:args.beam]
if args.objective == "maximin":
_save(ledger, args.market)
_save(ledger, args.market, {"selection_start": a, "selection_end": b,
"role": "development", "config": vars(args)})
return
# holdout, once, for the beam winners of each k
print("=== holdout of design winners ===")
print("=== historical validation of design winners (previously inspected data) ===")
ha, hb = periods["holdout"]
for c, st in best_overall:
s = sum(ranks[n] for n in c) / len(c)
hs = phase_avg(p, s, ha, hb, args.top_n, args.every)
print(f" {'+'.join(c):55s} D sh {st['sharpe']:.2f} cagr {st['cagr']*100:5.1f}% | "
f"H sh {hs['sharpe']:.2f} cagr {hs['cagr']*100:5.1f}% mdd {hs['mdd']*100:5.1f}%", flush=True)
_save(ledger, args.market)
_save(ledger, args.market, {"selection_start": a, "selection_end": b,
"role": "development", "config": vars(args)})
def _save(ledger: list[dict], market: str) -> None:
def _save(ledger: list[dict], market: str, protocol: dict) -> None:
os.makedirs(RESULTS, exist_ok=True)
path = os.path.join(RESULTS, f"trials_{market}.csv")
pd.DataFrame(ledger).to_csv(path, mode="a", header=not os.path.exists(path), index=False)
path = os.path.join(RESULTS, f"trials_{market}.jsonl")
append_trials(path, ledger, protocol)
if __name__ == "__main__":

45
research/lab/trials.py Normal file
View File

@@ -0,0 +1,45 @@
"""Append-only trial records; legacy CSVs are read without discarding wide rows."""
import csv
import json
from datetime import datetime, timezone
from pathlib import Path
import fcntl
def append_trials(path, rows, protocol):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
created = datetime.now(timezone.utc).isoformat()
with path.open('a') as fh:
fcntl.flock(fh, fcntl.LOCK_EX)
for row in rows:
record = {'schema': 1, 'created_at': created, 'protocol': protocol, 'result': row}
fh.write(json.dumps(record, allow_nan=False, default=float) + '\n')
def count_trials(directory, market):
"""Observed trials are a lower bound; unrecorded searches remain unknown."""
total = 0
paths = [Path(directory) / f'trials_{market}.csv', Path(directory) / f'trials_{market}.jsonl']
if market == 'us':
paths += [Path(directory) / 'trials_us_sweep.csv', Path(directory) / 'trials_us_sweep.jsonl']
for path in paths:
if not path.exists():
continue
with path.open() as fh:
if path.suffix == '.jsonl':
for line in fh:
if line.strip():
record = json.loads(line)
if record.get('schema') != 1:
raise ValueError(f'Unknown trial schema: {path}')
total += 1
else:
rows = csv.reader(fh)
next(rows, None)
for row in rows:
if row:
if len(row) < 2:
raise ValueError(f'Incomplete trial row: {path}')
total += 1
return total

View File

@@ -130,16 +130,13 @@ def main() -> None:
print(line(f" {c} 2006-", describe(rr[c].dropna())))
# deflated Sharpe: every configuration this and the previous round tried
n = 0
for f in ("trials_us.csv", "trials_us_sweep.csv"):
path = os.path.join(RESULTS, f)
if os.path.exists(path):
with open(path) as fh:
n += sum(1 for line in fh if not line.startswith(("combo,", "score,")))
n += 12 + 60 + 20 # rotation schemes, fund_eval variants, defensive variants
from research.lab.trials import count_trials
n = count_trials(RESULTS, "us")
if n < 1:
raise ValueError("A trial ledger is required for DSR")
r = monthly_blend(R.loc[SPAN[0]:SPAN[1]].fillna(0.0), three)
dsr = deflated_sharpe(r, n)
out["dsr"] = {"n_trials": n, **{k: float(v) for k, v in dsr.items()}}
out["dsr"] = {"observed_trials_lower_bound": n, "role": "development", **{k: float(v) for k, v in dsr.items()}}
print(f"\nDSR of the 3-sleeve book: {dsr['dsr']:.3f} (Sharpe {dsr['sr_annual']:.2f} vs "
f"expected max of {n} unskilled trials {dsr['sr0_annual']:.2f})")
print("\nyearly returns (3 sleeves vs SPY):")

View File

@@ -5,7 +5,7 @@ rules (PIT S&P 500 members, next-open fills, 10 bp costs, phase averaging).
PYTHONPATH=. uv run --with numba python -m research.lab.us_sweep --part construction
PYTHONPATH=. uv run --with numba python -m research.lab.us_sweep --part overlays
Every configuration is appended to data/research/results/trials_us_sweep.csv
Every configuration is appended to data/research/results/trials_us_sweep.jsonl
so the deflated Sharpe of the final pick can count them.
"""
@@ -100,11 +100,10 @@ def fmt(name, st) -> str:
def log_trials(rows, part):
os.makedirs(RESULTS, exist_ok=True)
path = os.path.join(RESULTS, "trials_us_sweep.csv")
df = pd.DataFrame([{k: v for k, v in r.items() if not k.startswith("_") and k != "blocks"} for r in rows])
df["part"] = part
df.to_csv(path, mode="a", header=not os.path.exists(path), index=False)
from research.lab.trials import append_trials
records = [{k: v for k, v in r.items() if not k.startswith("_") and k != "blocks"} for r in rows]
append_trials(os.path.join(RESULTS, "trials_us_sweep.jsonl"), records,
{"part": part, "role": "development; historical data already examined"})
def construction(p, S) -> list[dict]: