diff --git a/.gitignore b/.gitignore index e5185bb..3a67b32 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,9 @@ Thumbs.db data/.*.lock data/backup_*/ data/*_unavailable.json + +# Local secrets, logs and agent scratch output +.env +logs/ +memory/ +.tmp_finance/ diff --git a/precompute_backtest.py b/precompute_backtest.py new file mode 100644 index 0000000..2eb737d --- /dev/null +++ b/precompute_backtest.py @@ -0,0 +1,419 @@ +""" +Pre-compute backtest results for all strategies, for multiple capital scenarios. + +Each strategy is backtested ONCE per market with that market's live starting +capital (MONITOR_CAPITAL: US $10k, CN ¥70k); the +resulting equity series is then replayed with different cash-injection +schedules so we get multiple scenarios without re-running each strategy. + +Usage: + uv run python precompute_backtest.py [--output PATH] [--years N] + +Output JSON shape: + { + "computedAt": "...", + "scenarios": { + "lump_sum_10k": {label, capital, injectionSchedule, markets: {us, cn}}, + "dca_feb_aug": {label, capital, injectionSchedule, markets: {us, cn}} + } + } +""" + +import argparse +import json +import os +import sys +from datetime import datetime + +import numpy as np +import pandas as pd + +import data_manager +import metrics +import universe_history as uh +from bridge import ( + MIN_TRADE_VALUE, + STRATEGY_META, + _build_strategy_result, + _sanitize_for_json, + benchmark_curve, + compute_rebalance_trades, +) +from main import backtest +from trader import ( + strategy_extra_tickers, + strategy_supports_market, + ETF_STRATEGY_UNIVERSES, + MONITOR_CAPITAL, + STRATEGY_REGISTRY, + convert_quote_currency, + filter_tradable_tickers, + price_request, +) +from universe import UNIVERSES + + +# Initial capital per market, in local currency — the same amounts the live +# monitor starts with (US $10k, CN ¥70k ≈ $10k). Scenario amounts below are +# multiples of it, so both markets model the same real money. +MARKET_CAPITAL = dict(MONITOR_CAPITAL) + + +SCENARIOS = [ + { + "id": "lump_sum_10k", + "label": "起始资金一次性投入", + "description": "US $10,000 / CN ¥70,000 一次性投入,无后续追加", + "injection": None, + }, + { + "id": "dca_feb_aug", + "label": "起始资金 + 每年 2/8 月追加一半", + "description": "US $10,000 + 每年 2/8 月各 $5,000;CN ¥70,000 + 每年 2/8 月各 ¥35,000", + "injection": { + "amount_ratio": 0.5, + "frequency": "yearly_months", + "months": [2, 8], + "startYear": 1, + }, + }, +] + + +def _injection_for_market(injection: dict | None, capital: float) -> dict | None: + """Materialize a scenario's injection schedule in a market's currency.""" + if not injection: + return None + out = {k: v for k, v in injection.items() if k != "amount_ratio"} + out["amount"] = round(capital * injection["amount_ratio"], 2) + return out + + +def log(msg: str) -> None: + print(f"[precompute] {msg}", file=sys.stderr) + + +def _equity_series_from_curve(curve: list[dict]) -> pd.Series: + """Reconstruct a pandas Series from a precomputed equity curve list.""" + if not curve: + return pd.Series(dtype=float) + dates = pd.to_datetime([p["date"] for p in curve]) + values = [float(p["value"]) for p in curve] + return pd.Series(values, index=dates) + + +def run_market_raw( + market: str, + strategy_names: list[str], + years: int, + capital: float, + tx_cost: float, + fixed_fee: float, + fee_base: float = 0.0, + fee_per_share: float = 0.0, +) -> dict | None: + """Run all strategies once; return raw equity series + metadata per strategy.""" + stock_strategies = [] + etf_strategies = [] + for name in strategy_names: + base_name = name.removeprefix("sim_") + if base_name in ETF_STRATEGY_UNIVERSES: + etf_strategies.append(name) + else: + stock_strategies.append(name) + + raw_strategies: list[dict] = [] + benchmark_label = "SPY" + bench_curve = [] + start_date = None + end_date = None + + # --- Stock strategies --- + if stock_strategies: + universe = UNIVERSES[market] + tickers = universe["fetch"]() + benchmark_ticker = universe["benchmark"] + benchmark_label = universe["benchmark_label"] + + pit_intervals = None + if market == "us": + pit_intervals = uh.load_sp500_history() + historical_tickers = uh.all_tickers_ever(pit_intervals) + all_tickers = sorted(set(tickers + historical_tickers + [benchmark_ticker]) + | {t for n in stock_strategies for t in strategy_extra_tickers(n)}) + log(f"PIT universe: {len(all_tickers)} tickers") + else: + all_tickers = sorted(set(tickers + [benchmark_ticker]) + | {t for n in stock_strategies for t in strategy_extra_tickers(n)}) + log(f"WARNING: {market.upper()} lacks PIT history; using current constituents") + + log(f"Fetching price data for {len(all_tickers)} tickers...") + data = data_manager.update(market, all_tickers, with_open=False) + if isinstance(data, tuple): + data = data[0] + + if years: + cutoff = data.index[-1] - pd.DateOffset(years=years) + data = data[data.index >= cutoff] + + if pit_intervals is not None: + log("Applying PIT membership mask...") + data = uh.mask_prices(data, pit_intervals) + tickers = [ + t for t in data.columns + if t != benchmark_ticker and data[t].notna().any() + ] + else: + tickers = filter_tradable_tickers(data, tickers) + + log(f"Universe: {len(tickers)} stocks, {data.index[0].date()} to {data.index[-1].date()}") + + for name in stock_strategies: + log(f" Running: {name}") + try: + strategy = STRATEGY_REGISTRY[name]() + # Mixed stock+ETF strategies also read their fixed ETFs. + cols = tickers + [t for t in strategy_extra_tickers(name) + if t in data.columns and t not in tickers] + + equity = backtest( + strategy, data[cols], + initial_capital=capital, + transaction_cost=tx_cost, + fixed_fee=fixed_fee, + fee_base=fee_base, + fee_per_share=fee_per_share, + min_trade_value=MIN_TRADE_VALUE, + ) + + weights = strategy.generate_signals(data[cols]) + weights = weights.reindex(data[cols].index).fillna(0.0) + trade_events = compute_rebalance_trades(weights) + + raw_strategies.append({"name": name, "equity": equity, "trades": trade_events}) + log(f" OK: {name} (final: ${equity.iloc[-1]:,.0f})") + except Exception as e: + log(f" FAILED: {name}: {e}") + + # Benchmark on exactly the strategies' dates + bench_curve = benchmark_curve(data[benchmark_ticker], data.index, capital) + start_date = data.index[0].strftime("%Y-%m-%d") + end_date = data.index[-1].strftime("%Y-%m-%d") + + # --- ETF strategies --- + if etf_strategies: + for name in etf_strategies: + log(f" Running ETF strategy: {name}") + try: + base_name = name.removeprefix("sim_") + etf_tickers = ETF_STRATEGY_UNIVERSES[base_name] + all_etf = sorted(set(etf_tickers + ["SPY"])) + + etf_data = data_manager.update("etfs", price_request(market, all_etf), with_open=False) + if isinstance(etf_data, tuple): + etf_data = etf_data[0] + etf_data = convert_quote_currency(etf_data) + if years: + cutoff = etf_data.index[-1] - pd.DateOffset(years=years) + etf_data = etf_data[etf_data.index >= cutoff] + + strategy = STRATEGY_REGISTRY[name]() + + tradable = filter_tradable_tickers(etf_data, etf_tickers) + equity = backtest( + strategy, etf_data[tradable], + initial_capital=capital, + transaction_cost=tx_cost, + fixed_fee=fixed_fee, + fee_base=fee_base, + fee_per_share=fee_per_share, + min_trade_value=MIN_TRADE_VALUE, + ) + + weights = strategy.generate_signals(etf_data[tradable]) + weights = weights.reindex(etf_data[tradable].index).fillna(0.0) + trade_events = compute_rebalance_trades(weights) + + raw_strategies.append({"name": name, "equity": equity, "trades": trade_events}) + log(f" OK: {name} (final: ${equity.iloc[-1]:,.0f})") + except Exception as e: + log(f" FAILED: {name}: {e}") + + if not stock_strategies: + benchmark_label = "SPY (Benchmark)" + spy_data = data_manager.update("etfs", ["SPY"], with_open=False) + if isinstance(spy_data, tuple): + spy_data = spy_data[0] + if years: + cutoff = spy_data.index[-1] - pd.DateOffset(years=years) + spy_data = spy_data[spy_data.index >= cutoff] + if "SPY" in spy_data.columns: + bench_curve = benchmark_curve(spy_data["SPY"], spy_data.index, capital) + start_date = spy_data.index[0].strftime("%Y-%m-%d") + end_date = spy_data.index[-1].strftime("%Y-%m-%d") + + if not raw_strategies: + return None + + return { + "market": market, + "strategy_runs": raw_strategies, + "benchmark_label": benchmark_label, + "benchmark_curve": bench_curve, + "start_date": start_date, + "end_date": end_date, + "tx_cost": tx_cost, + "fixed_fee": fixed_fee, + } + + +def build_scenario_for_market( + raw: dict, + scenario: dict, + years: int, + capital: float, +) -> dict: + """Materialize a scenario (with optional injections) for one market.""" + injection_schedule = _injection_for_market(scenario.get("injection"), capital) + benchmark_curve = raw["benchmark_curve"] + benchmark_label = raw["benchmark_label"] + + strategy_results = [] + for run in raw["strategy_runs"]: + equity = run["equity"] + try: + result = _build_strategy_result( + run["name"], equity, run["trades"], injection_schedule, capital, + ) + strategy_results.append(result) + except Exception as e: + log(f" [scenario={scenario['id']}] FAILED {run['name']}: {e}") + + # Re-scale benchmark for DCA scenarios so the comparison stays apples-to-apples. + final_bench_curve = benchmark_curve + if injection_schedule and benchmark_curve: + bench_series = _equity_series_from_curve(benchmark_curve) + bench_returns = bench_series.pct_change().fillna(0.0) + from bridge import build_injection_schedule + from research.permanent_yearly import equity_with_cashflows + contribs, _ = build_injection_schedule(bench_series, injection_schedule) + bench_with_inj = equity_with_cashflows(bench_returns, contribs, capital) + final_bench_curve = [ + {"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)} + for d, v in bench_with_inj.items() + ] + + return { + "meta": { + "market": raw["market"], + "years": years, + "initialCapital": capital, + "startDate": raw["start_date"], + "endDate": raw["end_date"], + "txCost": raw["tx_cost"], + "fixedFee": raw["fixed_fee"], + "injectionSchedule": injection_schedule, + }, + "strategies": strategy_results, + "benchmark": { + "name": benchmark_label, + "equityCurve": final_bench_curve, + }, + } + + +def main(): + parser = argparse.ArgumentParser(description="Pre-compute backtest results") + parser.add_argument( + "--output", default=None, + help="Output directory (default: ../stock-agent/data/precomputed)", + ) + parser.add_argument( + "--years", type=int, default=10, + help="Backtest period in years (default: 10)", + ) + args = parser.parse_args() + + output_dir = args.output or os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "stock-agent", "data", "precomputed" + ) + os.makedirs(output_dir, exist_ok=True) + + us_strategies = [] + cn_strategies = [] + for name in STRATEGY_REGISTRY: + meta = STRATEGY_META.get(name, {"markets": ["us", "cn"]}) + markets = [m for m in meta.get("markets", ["us", "cn"]) if strategy_supports_market(name, m)] + if "us" in markets: + us_strategies.append(name) + if "cn" in markets: + cn_strategies.append(name) + + us_capital = MARKET_CAPITAL["us"] + cn_capital = MARKET_CAPITAL["cn"] + + # Run each market once (cached price data → fast subsequent runs across scenarios). + log("=" * 60) + log(f"US MARKET: {len(us_strategies)} strategies @ ${us_capital:,.0f}") + log("=" * 60) + # US fee schedule (IBKR-style tiered): + # fee = max(2.0, 1.88 + 0.009 * shares) + us_raw = run_market_raw( + "us", us_strategies, + years=args.years, capital=us_capital, + tx_cost=0.001, fixed_fee=2.0, + fee_base=1.88, fee_per_share=0.009, + ) + + log("=" * 60) + log(f"CN MARKET: {len(cn_strategies)} strategies @ ¥{cn_capital:,.0f}") + log("=" * 60) + cn_raw = run_market_raw( + "cn", cn_strategies, + years=args.years, capital=cn_capital, + tx_cost=0.001, fixed_fee=5.0, + ) + + # Materialize every scenario × market by applying its injection schedule. + scenarios_out: dict[str, dict] = {} + for scenario in SCENARIOS: + log(f"--- scenario: {scenario['id']} ---") + scenarios_out[scenario["id"]] = { + "id": scenario["id"], + "label": scenario["label"], + "description": scenario["description"], + "capital": us_capital, + "capitalByMarket": {"us": us_capital, "cn": cn_capital}, + "injectionSchedule": _injection_for_market(scenario["injection"], us_capital), + "markets": {}, + } + if us_raw: + scenarios_out[scenario["id"]]["markets"]["us"] = build_scenario_for_market( + us_raw, scenario, args.years, us_capital + ) + if cn_raw: + scenarios_out[scenario["id"]]["markets"]["cn"] = build_scenario_for_market( + cn_raw, scenario, args.years, cn_capital + ) + + output = { + "computedAt": datetime.now().isoformat(), + "config": {"years": args.years, "initialCapital": MARKET_CAPITAL}, + "scenarios": scenarios_out, + } + + output_path = os.path.join(output_dir, "backtest_results.json") + with open(output_path, "w") as f: + json.dump(_sanitize_for_json(output), f, default=str) + + file_size = os.path.getsize(output_path) / (1024 * 1024) + log("=" * 60) + log(f"DONE: Written to {output_path} ({file_size:.1f} MB)") + for sid, sc in scenarios_out.items(): + for mk, mv in sc["markets"].items(): + log(f" [{sid}] {mk.upper()}: {len(mv['strategies'])} strategies") + log("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_daily.py b/scripts/run_daily.py new file mode 100644 index 0000000..2807b71 --- /dev/null +++ b/scripts/run_daily.py @@ -0,0 +1,116 @@ +"""Daily live-trading update for all registered strategies. + +Iterates STRATEGY_REGISTRY and calls trader.cmd_auto for each, sharing +fetched price panels across strategies (we still call cmd_auto so the +state-file format stays consistent with manual runs). + +Usage: + uv run python scripts/run_daily.py [--market us|cn|all] [--capital 10000] + [--only NAME ...] [--skip NAME ...] + +Designed for cron: + # 5pm ET weekdays, after US close + 0 17 * * 1-5 cd /path/to/quant && uv run python scripts/run_daily.py --market us + # 4pm CST weekdays, after CN close + 0 16 * * 1-5 cd /path/to/quant && uv run python scripts/run_daily.py --market cn +""" +from __future__ import annotations + +import argparse +import os +import sys +import time +import types +from datetime import datetime + +# Make the project root importable when invoked as `python scripts/run_daily.py` +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + +# We change into the project root so trader's relative `data/` paths resolve. +os.chdir(PROJECT_ROOT) + +from trader import ( # noqa: E402 + ETF_STRATEGY_UNIVERSES, + STRATEGY_REGISTRY, + cmd_auto, +) +from bridge import STRATEGY_META # noqa: E402 + + +def _strategy_markets(name: str) -> list[str]: + """Return the markets a strategy targets (per its META declaration).""" + meta = STRATEGY_META.get(name, {"markets": ["us"]}) + return list(meta.get("markets") or ["us"]) + + +def _make_args(market: str, name: str, capital: float) -> argparse.Namespace: + """Build the argparse namespace cmd_auto expects.""" + return argparse.Namespace( + market=market, + strategy=name, + capital=capital, + tx_cost=0.001, + fixed_fee=0.0, # cmd_auto falls back to MARKET_FEES + tier + integer_shares=True, # match the existing live state convention + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Daily live-trading update") + parser.add_argument("--market", default="all", + help="Which market(s): 'us', 'cn', 'all' (default: all)") + parser.add_argument("--capital", type=float, default=10_000, + help="Initial capital for any newly-initialized strategy (default: 10000)") + parser.add_argument("--only", nargs="+", default=None, + help="If set, only run these strategy names") + parser.add_argument("--skip", nargs="+", default=None, + help="Strategy names to skip (e.g. broken ones)") + parser.add_argument("--dry-run", action="store_true", + help="Print what would be run without executing") + args = parser.parse_args() + + markets = ["us", "cn"] if args.market == "all" else [args.market] + only = set(args.only) if args.only else None + skip = set(args.skip) if args.skip else set() + + plan: list[tuple[str, str]] = [] + for name in STRATEGY_REGISTRY: + if only and name not in only: + continue + if name in skip: + continue + for mk in _strategy_markets(name): + if mk in markets: + plan.append((mk, name)) + + print(f"[run_daily] {datetime.now().isoformat(timespec='seconds')} " + f"will process {len(plan)} (market, strategy) pairs") + + if args.dry_run: + for mk, name in plan: + print(f" [DRY] {mk:<3} {name}") + return 0 + + failures = [] + t0 = time.time() + for mk, name in plan: + ns = _make_args(mk, name, args.capital) + try: + cmd_auto(ns) + except SystemExit: + raise + except Exception as e: # noqa: BLE001 + failures.append((mk, name, repr(e))) + print(f"[run_daily] FAILED {mk} {name}: {e}", file=sys.stderr) + + elapsed = time.time() - t0 + print(f"[run_daily] done in {elapsed:.1f}s — {len(plan) - len(failures)} ok, {len(failures)} failed") + for mk, name, err in failures: + print(f" fail {mk} {name}: {err}", file=sys.stderr) + return 0 if not failures else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_daily.sh b/scripts/run_daily.sh new file mode 100755 index 0000000..4da96d2 --- /dev/null +++ b/scripts/run_daily.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Daily live-trading update entrypoint for cron. +# +# Examples: +# 0 17 * * 1-5 /path/to/quant/scripts/run_daily.sh us >> /var/log/quant_us.log 2>&1 +# 0 16 * * 1-5 /path/to/quant/scripts/run_daily.sh cn >> /var/log/quant_cn.log 2>&1 +# +# The script changes into the quant project root and calls run_daily.py via uv. +# Defaults to "all" markets when no argument is passed. + +set -euo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" + +cd "$PROJECT_ROOT" + +MARKET="${1:-all}" +shift || true + +exec uv run python scripts/run_daily.py --market "$MARKET" "$@" diff --git a/scripts/start-monitor.sh b/scripts/start-monitor.sh new file mode 100755 index 0000000..3cbae42 --- /dev/null +++ b/scripts/start-monitor.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Long-running live-trader monitor (US + CN, all strategies). +# Started by the quant-monitor.service systemd --user unit; replaces the +# old "tmux new -s quant; uv run python trader.py monitor" workflow. +set -euo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" +cd "$PROJECT_ROOT" + +# uv lives in ~/.local/bin, which is not on systemd's default PATH. +export PATH="/home/gahow/.local/bin:$PATH" + +# Market data is fetched through the lab proxy (same as interactive `proxyOn`). +# QUANT_MARKET_PROXY lives in the untracked repo-root .env. +if [[ -f .env ]]; then + set -a; source .env; set +a +fi +if [[ -n "${QUANT_MARKET_PROXY:-}" ]]; then + export http_proxy="$QUANT_MARKET_PROXY" https_proxy="$QUANT_MARKET_PROXY" all_proxy="$QUANT_MARKET_PROXY" +fi + +exec uv run python trader.py monitor