fix: unify backtest execution and add forward validation candidates
Share point-in-time signals and a cash-funded order ledger across backtests and trading. Correct eligibility, missing-price fills, quality factors, regime inputs, warmup windows, and cash-injection accounting. Add reproducible registered-strategy comparisons and paired forward candidates with a 20% resize band. Validation: 125 Python tests, 23 web tests, TypeScript/Vite build, 87 strategy causality checks, and real-data target/order/equity parity for 10 strategies.
This commit is contained in:
@@ -20,6 +20,7 @@ from research.lab.engine import Costs, simulate
|
||||
from research.lab.factor_scan import RESULTS, load
|
||||
from research.lab.report import deflated_sharpe, stats, yearly
|
||||
from research.lab.stepwise import SUBPERIODS
|
||||
from research.lab.trials import count_trials
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
@@ -81,7 +82,15 @@ def main() -> None:
|
||||
ap.add_argument("--market", default="us")
|
||||
ap.add_argument("--add", nargs="*", default=[], help="extra 'name=combo|top|every|buffer'")
|
||||
ap.add_argument("--n-trials", type=int, default=0, help="override trial count for DSR")
|
||||
ap.add_argument("--registered", nargs="+", help="Validate registered strategies using the production ledger")
|
||||
ap.add_argument("--start")
|
||||
ap.add_argument("--end")
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
if args.registered:
|
||||
from research.lab.registered import evaluate_registered
|
||||
evaluate_registered(args)
|
||||
return
|
||||
mk = "us" if args.market == "us" else "cn"
|
||||
p, _ = load(args.market)
|
||||
cands = dict(CANDIDATES[mk])
|
||||
@@ -89,8 +98,9 @@ def main() -> None:
|
||||
name, rest = spec.split("=", 1)
|
||||
combo, top, every, buf = rest.split("|")
|
||||
cands[name] = (combo, dict(top_n=int(top), every=int(every), buffer=float(buf)))
|
||||
trials_path = os.path.join(RESULTS, f"trials_{args.market}.csv")
|
||||
n_trials = args.n_trials or (len(pd.read_csv(trials_path)) if os.path.exists(trials_path) else 100)
|
||||
n_trials = args.n_trials or count_trials(RESULTS, args.market)
|
||||
if n_trials < 1:
|
||||
raise ValueError("DSR needs a trial ledger or an explicit --n-trials count")
|
||||
full = (SUBPERIODS[mk][0][0], WINDOW[1])
|
||||
out = {}
|
||||
fee, cap = LIVE_FEES[mk]
|
||||
@@ -115,7 +125,8 @@ def main() -> None:
|
||||
expo = (0.5 + 0.25 * slow + 0.25 * fast).where(slow + fast < 2, 1.0)
|
||||
res["window_trend_overlay"] = summarize(phase_runs(p, s, *WINDOW, exposure=expo, **kw), *WINDOW)
|
||||
dsr = deflated_sharpe(res["full"]["_rets"], n_trials)
|
||||
res["dsr_full"] = dsr
|
||||
res["dsr_full"] = {**dsr, "observed_trials_lower_bound": n_trials,
|
||||
"evidence_role": "development, not independent OOS"}
|
||||
out[name] = res
|
||||
w, f = res["window"], res["full"]
|
||||
print(f"{name:42s} 2016-26: CAGR {w['cagr']*100:5.1f}% [{w['cagr_phase_min']*100:4.1f},{w['cagr_phase_max']*100:4.1f}] "
|
||||
|
||||
111
research/lab/registered.py
Normal file
111
research/lab/registered.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""Fixed-budget validation of registered strategies on the shared execution engine."""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import metrics
|
||||
import data_manager
|
||||
from backtest_runner import (ENGINE_VERSION, load_stock_panels, prepare_strategy, replay_prepared)
|
||||
from trader import make_strategy
|
||||
from strategies.robust_momentum import RobustMomentumStrategy
|
||||
from research.lab.trials import append_trials
|
||||
|
||||
|
||||
def _stats(equity):
|
||||
return {k: v for k, v in metrics.raw_summary(equity).items()
|
||||
if k in ('annualizedReturn', 'maxDrawdown', 'sharpeRatio', 'annualizedVolatility')}
|
||||
|
||||
|
||||
def evaluate_registered(args):
|
||||
market = 'cn' if args.market.startswith('cn') else 'us'
|
||||
destination = Path(args.output or f'data/research/results/registered_{market}')
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
panels = load_stock_panels(market, args.registered, None, refresh=False)
|
||||
end = args.end or str(panels['panels']['close'].index[-1].date())
|
||||
start = args.start or '2018-01-01'
|
||||
profiles = ['baseline', 'resize20', 'buffer3'] + (['cash_shy'] if market == 'us' else [])
|
||||
protocol = {
|
||||
'created_at': datetime.now(timezone.utc).isoformat(), 'engine': ENGINE_VERSION,
|
||||
'market': market, 'strategies': args.registered, 'start': start, 'end': end,
|
||||
'profiles': profiles, 'one_way_costs': [.001, .0025], 'capital': 10000 if market == 'us' else 70000,
|
||||
'selection_rule': 'CAGR improves at both costs; drawdown deterioration <= 2pp; improves in >=2/3 time blocks',
|
||||
'role': 'historical development comparison; not independent OOS; do not auto-promote',
|
||||
'source_hashes': {str(p): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in [Path('execution.py'), Path('signals.py'), Path('backtest_runner.py')]
|
||||
+ sorted(Path('strategies').glob('*.py'))},
|
||||
'data_hashes': {str(p): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in [Path('data') / f'{market}.csv', Path('data') / f'{market}_open.csv',
|
||||
Path('data/sp500_history.json' if market == 'us' else 'data/csi300_history.json')]},
|
||||
}
|
||||
protocol_path = destination / 'protocol.json'
|
||||
if protocol_path.exists():
|
||||
raise FileExistsError(f'Choose a new --output to preserve the completed study: {destination}')
|
||||
protocol_path.write_text(json.dumps(protocol, indent=2))
|
||||
rows, verdicts = [], []
|
||||
fees = {'tx_cost': .001, 'fixed_fee': 2 if market == 'us' else 5,
|
||||
'fee_base': 1.88 if market == 'us' else 0, 'fee_per_share': .009 if market == 'us' else 0}
|
||||
for name in args.registered:
|
||||
base_strategy = make_strategy(name, market)
|
||||
base_plan = prepare_strategy(name, market, panels['panels'], panels['stocks'], member=panels['member'],
|
||||
start=start, end=end, strategy=base_strategy)
|
||||
if base_plan['warmup_rows'] < 315:
|
||||
raise ValueError('Registered comparison requires >=315 pre-window sessions; move --start or add history')
|
||||
dates = base_plan['prices'].index
|
||||
cuts = [0, len(dates)//3, 2*len(dates)//3, len(dates)]
|
||||
for profile in profiles:
|
||||
plan, band = base_plan, .1
|
||||
if profile == 'resize20':
|
||||
band = .2
|
||||
elif profile == 'buffer3':
|
||||
if not isinstance(base_strategy, RobustMomentumStrategy):
|
||||
continue
|
||||
changed = copy.deepcopy(base_strategy); changed.buffer = 3.0
|
||||
plan = prepare_strategy(name, market, panels['panels'], panels['stocks'], member=panels['member'],
|
||||
start=start, end=end, strategy=changed)
|
||||
elif profile == 'cash_shy':
|
||||
if 'SHY' not in panels['panels']['close']:
|
||||
raise ValueError('SHY bars are required for a traded cash sleeve')
|
||||
plan = dict(base_plan)
|
||||
plan['prices'] = base_plan['prices'].copy()
|
||||
plan['prices']['SHY'] = panels['panels']['close']['SHY'].reindex(dates)
|
||||
if base_plan.get('volume') is not None:
|
||||
plan['volume'] = base_plan['volume'].copy()
|
||||
plan['volume']['SHY'] = panels['panels']['volume']['SHY'].reindex(dates)
|
||||
target = base_plan['targets'].reindex(columns=plan['prices'].columns, fill_value=0).copy()
|
||||
target['SHY'] += (1-target.sum(axis=1)).clip(lower=0)
|
||||
plan['targets'] = target
|
||||
for cost in protocol['one_way_costs']:
|
||||
run = replay_prepared(plan, protocol['capital'], **{**fees, 'tx_cost': cost}, resize_band=band)
|
||||
row = {'strategy': name, 'profile': profile, 'cost': cost, **_stats(run['equity']),
|
||||
'orders': len(run['trade_log']), 'fees': sum(t['commission'] for t in run['trade_log']),
|
||||
'minimum_cash': float(run['cash'].min()), 'blocks': []}
|
||||
for a, b in zip(cuts[:-1], cuts[1:]):
|
||||
row['blocks'].append(_stats(run['equity'].iloc[a:b+1]))
|
||||
rows.append(row)
|
||||
append_trials(destination/'trials.jsonl', [row], protocol)
|
||||
print(name, profile, cost, f"CAGR {row['annualizedReturn']:.2%}",
|
||||
f"DD {row['maxDrawdown']:.2%}", flush=True)
|
||||
baseline = {r['cost']: r for r in rows if r['strategy'] == name and r['profile'] == 'baseline'}
|
||||
for profile in profiles[1:]:
|
||||
candidates = [r for r in rows if r['strategy'] == name and r['profile'] == profile]
|
||||
if not candidates:
|
||||
continue
|
||||
passed = all(r['annualizedReturn'] > baseline[r['cost']]['annualizedReturn']
|
||||
and r['maxDrawdown'] >= baseline[r['cost']]['maxDrawdown']-.02
|
||||
and sum(a['annualizedReturn'] > b['annualizedReturn']
|
||||
for a,b in zip(r['blocks'],baseline[r['cost']]['blocks'])) >= 2
|
||||
for r in candidates)
|
||||
verdicts.append({'strategy': name, 'profile': profile,
|
||||
'verdict': 'candidate_for_forward_only' if passed else 'retain_baseline'})
|
||||
summary = {'protocol': protocol, 'comparisons': rows, 'decisions': verdicts,
|
||||
'boundary': 'Missing historical constituents and adjusted-share execution remain limitations.'}
|
||||
(destination/'results.json').write_text(json.dumps(summary, indent=2, allow_nan=False))
|
||||
pd.DataFrame([{k:v for k,v in r.items() if k!='blocks'} for r in rows]).to_csv(destination/'results.csv',index=False)
|
||||
print('Saved',destination/'results.json',flush=True)
|
||||
return summary
|
||||
@@ -1,11 +1,12 @@
|
||||
"""
|
||||
Forward stepwise selection of equal-weight rank composites, scored on the
|
||||
DESIGN period only; the holdout is evaluated once for the final picks.
|
||||
DESIGN period only; historical validation is reported separately. Previously inspected history
|
||||
is not a fresh out-of-sample test.
|
||||
|
||||
PYTHONPATH=. uv run python -m research.lab.stepwise --market us --max-k 4
|
||||
|
||||
Every evaluated composite is appended to the trial ledger
|
||||
(data/research/results/trials_{market}.csv) so deflated Sharpe can count them.
|
||||
(data/research/results/trials_{market}.jsonl) so deflated Sharpe can count them.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -18,6 +19,7 @@ import pandas as pd
|
||||
|
||||
from research.lab.factor_scan import RESULTS, load, phase_avg
|
||||
from research.lab.factors import FACTORS, zrank
|
||||
from research.lab.trials import append_trials
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
@@ -55,7 +57,8 @@ def main() -> None:
|
||||
print(f"{len(names)} factor ranks in {time.time() - t:.0f}s", flush=True)
|
||||
a, b = periods["design"]
|
||||
ledger = []
|
||||
subs = SUBPERIODS[mk]
|
||||
subs = [(max(sa, a), min(sb, b)) for sa, sb in SUBPERIODS[mk]
|
||||
if max(sa, a) <= min(sb, b)]
|
||||
|
||||
def evaluate(combo: tuple[str, ...]) -> dict:
|
||||
s = sum(ranks[n] for n in combo) / len(combo)
|
||||
@@ -93,23 +96,25 @@ def main() -> None:
|
||||
best_overall += ranked[:args.beam]
|
||||
|
||||
if args.objective == "maximin":
|
||||
_save(ledger, args.market)
|
||||
_save(ledger, args.market, {"selection_start": a, "selection_end": b,
|
||||
"role": "development", "config": vars(args)})
|
||||
return
|
||||
# holdout, once, for the beam winners of each k
|
||||
print("=== holdout of design winners ===")
|
||||
print("=== historical validation of design winners (previously inspected data) ===")
|
||||
ha, hb = periods["holdout"]
|
||||
for c, st in best_overall:
|
||||
s = sum(ranks[n] for n in c) / len(c)
|
||||
hs = phase_avg(p, s, ha, hb, args.top_n, args.every)
|
||||
print(f" {'+'.join(c):55s} D sh {st['sharpe']:.2f} cagr {st['cagr']*100:5.1f}% | "
|
||||
f"H sh {hs['sharpe']:.2f} cagr {hs['cagr']*100:5.1f}% mdd {hs['mdd']*100:5.1f}%", flush=True)
|
||||
_save(ledger, args.market)
|
||||
_save(ledger, args.market, {"selection_start": a, "selection_end": b,
|
||||
"role": "development", "config": vars(args)})
|
||||
|
||||
|
||||
def _save(ledger: list[dict], market: str) -> None:
|
||||
def _save(ledger: list[dict], market: str, protocol: dict) -> None:
|
||||
os.makedirs(RESULTS, exist_ok=True)
|
||||
path = os.path.join(RESULTS, f"trials_{market}.csv")
|
||||
pd.DataFrame(ledger).to_csv(path, mode="a", header=not os.path.exists(path), index=False)
|
||||
path = os.path.join(RESULTS, f"trials_{market}.jsonl")
|
||||
append_trials(path, ledger, protocol)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
45
research/lab/trials.py
Normal file
45
research/lab/trials.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""Append-only trial records; legacy CSVs are read without discarding wide rows."""
|
||||
import csv
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import fcntl
|
||||
|
||||
|
||||
def append_trials(path, rows, protocol):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
created = datetime.now(timezone.utc).isoformat()
|
||||
with path.open('a') as fh:
|
||||
fcntl.flock(fh, fcntl.LOCK_EX)
|
||||
for row in rows:
|
||||
record = {'schema': 1, 'created_at': created, 'protocol': protocol, 'result': row}
|
||||
fh.write(json.dumps(record, allow_nan=False, default=float) + '\n')
|
||||
|
||||
|
||||
def count_trials(directory, market):
|
||||
"""Observed trials are a lower bound; unrecorded searches remain unknown."""
|
||||
total = 0
|
||||
paths = [Path(directory) / f'trials_{market}.csv', Path(directory) / f'trials_{market}.jsonl']
|
||||
if market == 'us':
|
||||
paths += [Path(directory) / 'trials_us_sweep.csv', Path(directory) / 'trials_us_sweep.jsonl']
|
||||
for path in paths:
|
||||
if not path.exists():
|
||||
continue
|
||||
with path.open() as fh:
|
||||
if path.suffix == '.jsonl':
|
||||
for line in fh:
|
||||
if line.strip():
|
||||
record = json.loads(line)
|
||||
if record.get('schema') != 1:
|
||||
raise ValueError(f'Unknown trial schema: {path}')
|
||||
total += 1
|
||||
else:
|
||||
rows = csv.reader(fh)
|
||||
next(rows, None)
|
||||
for row in rows:
|
||||
if row:
|
||||
if len(row) < 2:
|
||||
raise ValueError(f'Incomplete trial row: {path}')
|
||||
total += 1
|
||||
return total
|
||||
@@ -130,16 +130,13 @@ def main() -> None:
|
||||
print(line(f" {c} 2006-", describe(rr[c].dropna())))
|
||||
|
||||
# deflated Sharpe: every configuration this and the previous round tried
|
||||
n = 0
|
||||
for f in ("trials_us.csv", "trials_us_sweep.csv"):
|
||||
path = os.path.join(RESULTS, f)
|
||||
if os.path.exists(path):
|
||||
with open(path) as fh:
|
||||
n += sum(1 for line in fh if not line.startswith(("combo,", "score,")))
|
||||
n += 12 + 60 + 20 # rotation schemes, fund_eval variants, defensive variants
|
||||
from research.lab.trials import count_trials
|
||||
n = count_trials(RESULTS, "us")
|
||||
if n < 1:
|
||||
raise ValueError("A trial ledger is required for DSR")
|
||||
r = monthly_blend(R.loc[SPAN[0]:SPAN[1]].fillna(0.0), three)
|
||||
dsr = deflated_sharpe(r, n)
|
||||
out["dsr"] = {"n_trials": n, **{k: float(v) for k, v in dsr.items()}}
|
||||
out["dsr"] = {"observed_trials_lower_bound": n, "role": "development", **{k: float(v) for k, v in dsr.items()}}
|
||||
print(f"\nDSR of the 3-sleeve book: {dsr['dsr']:.3f} (Sharpe {dsr['sr_annual']:.2f} vs "
|
||||
f"expected max of {n} unskilled trials {dsr['sr0_annual']:.2f})")
|
||||
print("\nyearly returns (3 sleeves vs SPY):")
|
||||
|
||||
@@ -5,7 +5,7 @@ rules (PIT S&P 500 members, next-open fills, 10 bp costs, phase averaging).
|
||||
PYTHONPATH=. uv run --with numba python -m research.lab.us_sweep --part construction
|
||||
PYTHONPATH=. uv run --with numba python -m research.lab.us_sweep --part overlays
|
||||
|
||||
Every configuration is appended to data/research/results/trials_us_sweep.csv
|
||||
Every configuration is appended to data/research/results/trials_us_sweep.jsonl
|
||||
so the deflated Sharpe of the final pick can count them.
|
||||
"""
|
||||
|
||||
@@ -100,11 +100,10 @@ def fmt(name, st) -> str:
|
||||
|
||||
|
||||
def log_trials(rows, part):
|
||||
os.makedirs(RESULTS, exist_ok=True)
|
||||
path = os.path.join(RESULTS, "trials_us_sweep.csv")
|
||||
df = pd.DataFrame([{k: v for k, v in r.items() if not k.startswith("_") and k != "blocks"} for r in rows])
|
||||
df["part"] = part
|
||||
df.to_csv(path, mode="a", header=not os.path.exists(path), index=False)
|
||||
from research.lab.trials import append_trials
|
||||
records = [{k: v for k, v in r.items() if not k.startswith("_") and k != "blocks"} for r in rows]
|
||||
append_trials(os.path.join(RESULTS, "trials_us_sweep.jsonl"), records,
|
||||
{"part": part, "role": "development; historical data already examined"})
|
||||
|
||||
|
||||
def construction(p, S) -> list[dict]:
|
||||
|
||||
Reference in New Issue
Block a user