chore: track precompute and ops scripts; move monitor proxy into .env

The monitor start script used to embed the proxy URL with credentials,
which kept scripts/ out of git. It now reads QUANT_MARKET_PROXY from the
untracked repo-root .env.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.gitignore

@@ -49,3 +49,9 @@ Thumbs.db
 data/.*.lock
 data/backup_*/
 data/*_unavailable.json
+
+# Local secrets, logs and agent scratch output
+.env
+logs/
+memory/
+.tmp_finance/

precompute_backtest.py

@@ -0,0 +1,419 @@
+"""
+Pre-compute backtest results for all strategies, for multiple capital scenarios.
+
+Each strategy is backtested ONCE per market with that market's live starting
+capital (MONITOR_CAPITAL: US $10k, CN ¥70k); the
+resulting equity series is then replayed with different cash-injection
+schedules so we get multiple scenarios without re-running each strategy.
+
+Usage:
+    uv run python precompute_backtest.py [--output PATH] [--years N]
+
+Output JSON shape:
+    {
+        "computedAt": "...",
+        "scenarios": {
+            "lump_sum_10k": {label, capital, injectionSchedule, markets: {us, cn}},
+            "dca_feb_aug":  {label, capital, injectionSchedule, markets: {us, cn}}
+        }
+    }
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+
+import data_manager
+import metrics
+import universe_history as uh
+from bridge import (
+    MIN_TRADE_VALUE,
+    STRATEGY_META,
+    _build_strategy_result,
+    _sanitize_for_json,
+    benchmark_curve,
+    compute_rebalance_trades,
+)
+from main import backtest
+from trader import (
+    strategy_extra_tickers,
+    strategy_supports_market,
+    ETF_STRATEGY_UNIVERSES,
+    MONITOR_CAPITAL,
+    STRATEGY_REGISTRY,
+    convert_quote_currency,
+    filter_tradable_tickers,
+    price_request,
+)
+from universe import UNIVERSES
+
+
+# Initial capital per market, in local currency — the same amounts the live
+# monitor starts with (US $10k, CN ¥70k ≈ $10k). Scenario amounts below are
+# multiples of it, so both markets model the same real money.
+MARKET_CAPITAL = dict(MONITOR_CAPITAL)
+
+
+SCENARIOS = [
+    {
+        "id": "lump_sum_10k",
+        "label": "起始资金一次性投入",
+        "description": "US $10,000 / CN ¥70,000 一次性投入，无后续追加",
+        "injection": None,
+    },
+    {
+        "id": "dca_feb_aug",
+        "label": "起始资金 + 每年 2/8 月追加一半",
+        "description": "US $10,000 + 每年 2/8 月各 $5,000；CN ¥70,000 + 每年 2/8 月各 ¥35,000",
+        "injection": {
+            "amount_ratio": 0.5,
+            "frequency": "yearly_months",
+            "months": [2, 8],
+            "startYear": 1,
+        },
+    },
+]
+
+
+def _injection_for_market(injection: dict | None, capital: float) -> dict | None:
+    """Materialize a scenario's injection schedule in a market's currency."""
+    if not injection:
+        return None
+    out = {k: v for k, v in injection.items() if k != "amount_ratio"}
+    out["amount"] = round(capital * injection["amount_ratio"], 2)
+    return out
+
+
+def log(msg: str) -> None:
+    print(f"[precompute] {msg}", file=sys.stderr)
+
+
+def _equity_series_from_curve(curve: list[dict]) -> pd.Series:
+    """Reconstruct a pandas Series from a precomputed equity curve list."""
+    if not curve:
+        return pd.Series(dtype=float)
+    dates = pd.to_datetime([p["date"] for p in curve])
+    values = [float(p["value"]) for p in curve]
+    return pd.Series(values, index=dates)
+
+
+def run_market_raw(
+    market: str,
+    strategy_names: list[str],
+    years: int,
+    capital: float,
+    tx_cost: float,
+    fixed_fee: float,
+    fee_base: float = 0.0,
+    fee_per_share: float = 0.0,
+) -> dict | None:
+    """Run all strategies once; return raw equity series + metadata per strategy."""
+    stock_strategies = []
+    etf_strategies = []
+    for name in strategy_names:
+        base_name = name.removeprefix("sim_")
+        if base_name in ETF_STRATEGY_UNIVERSES:
+            etf_strategies.append(name)
+        else:
+            stock_strategies.append(name)
+
+    raw_strategies: list[dict] = []
+    benchmark_label = "SPY"
+    bench_curve = []
+    start_date = None
+    end_date = None
+
+    # --- Stock strategies ---
+    if stock_strategies:
+        universe = UNIVERSES[market]
+        tickers = universe["fetch"]()
+        benchmark_ticker = universe["benchmark"]
+        benchmark_label = universe["benchmark_label"]
+
+        pit_intervals = None
+        if market == "us":
+            pit_intervals = uh.load_sp500_history()
+            historical_tickers = uh.all_tickers_ever(pit_intervals)
+            all_tickers = sorted(set(tickers + historical_tickers + [benchmark_ticker])
+                                 | {t for n in stock_strategies for t in strategy_extra_tickers(n)})
+            log(f"PIT universe: {len(all_tickers)} tickers")
+        else:
+            all_tickers = sorted(set(tickers + [benchmark_ticker])
+                                 | {t for n in stock_strategies for t in strategy_extra_tickers(n)})
+            log(f"WARNING: {market.upper()} lacks PIT history; using current constituents")
+
+        log(f"Fetching price data for {len(all_tickers)} tickers...")
+        data = data_manager.update(market, all_tickers, with_open=False)
+        if isinstance(data, tuple):
+            data = data[0]
+
+        if years:
+            cutoff = data.index[-1] - pd.DateOffset(years=years)
+            data = data[data.index >= cutoff]
+
+        if pit_intervals is not None:
+            log("Applying PIT membership mask...")
+            data = uh.mask_prices(data, pit_intervals)
+            tickers = [
+                t for t in data.columns
+                if t != benchmark_ticker and data[t].notna().any()
+            ]
+        else:
+            tickers = filter_tradable_tickers(data, tickers)
+
+        log(f"Universe: {len(tickers)} stocks, {data.index[0].date()} to {data.index[-1].date()}")
+
+        for name in stock_strategies:
+            log(f"  Running: {name}")
+            try:
+                strategy = STRATEGY_REGISTRY[name]()
+                # Mixed stock+ETF strategies also read their fixed ETFs.
+                cols = tickers + [t for t in strategy_extra_tickers(name)
+                                  if t in data.columns and t not in tickers]
+
+                equity = backtest(
+                    strategy, data[cols],
+                    initial_capital=capital,
+                    transaction_cost=tx_cost,
+                    fixed_fee=fixed_fee,
+                    fee_base=fee_base,
+                    fee_per_share=fee_per_share,
+                    min_trade_value=MIN_TRADE_VALUE,
+                )
+
+                weights = strategy.generate_signals(data[cols])
+                weights = weights.reindex(data[cols].index).fillna(0.0)
+                trade_events = compute_rebalance_trades(weights)
+
+                raw_strategies.append({"name": name, "equity": equity, "trades": trade_events})
+                log(f"    OK: {name} (final: ${equity.iloc[-1]:,.0f})")
+            except Exception as e:
+                log(f"    FAILED: {name}: {e}")
+
+        # Benchmark on exactly the strategies' dates
+        bench_curve = benchmark_curve(data[benchmark_ticker], data.index, capital)
+        start_date = data.index[0].strftime("%Y-%m-%d")
+        end_date = data.index[-1].strftime("%Y-%m-%d")
+
+    # --- ETF strategies ---
+    if etf_strategies:
+        for name in etf_strategies:
+            log(f"  Running ETF strategy: {name}")
+            try:
+                base_name = name.removeprefix("sim_")
+                etf_tickers = ETF_STRATEGY_UNIVERSES[base_name]
+                all_etf = sorted(set(etf_tickers + ["SPY"]))
+
+                etf_data = data_manager.update("etfs", price_request(market, all_etf), with_open=False)
+                if isinstance(etf_data, tuple):
+                    etf_data = etf_data[0]
+                etf_data = convert_quote_currency(etf_data)
+                if years:
+                    cutoff = etf_data.index[-1] - pd.DateOffset(years=years)
+                    etf_data = etf_data[etf_data.index >= cutoff]
+
+                strategy = STRATEGY_REGISTRY[name]()
+
+                tradable = filter_tradable_tickers(etf_data, etf_tickers)
+                equity = backtest(
+                    strategy, etf_data[tradable],
+                    initial_capital=capital,
+                    transaction_cost=tx_cost,
+                    fixed_fee=fixed_fee,
+                    fee_base=fee_base,
+                    fee_per_share=fee_per_share,
+                    min_trade_value=MIN_TRADE_VALUE,
+                )
+
+                weights = strategy.generate_signals(etf_data[tradable])
+                weights = weights.reindex(etf_data[tradable].index).fillna(0.0)
+                trade_events = compute_rebalance_trades(weights)
+
+                raw_strategies.append({"name": name, "equity": equity, "trades": trade_events})
+                log(f"    OK: {name} (final: ${equity.iloc[-1]:,.0f})")
+            except Exception as e:
+                log(f"    FAILED: {name}: {e}")
+
+        if not stock_strategies:
+            benchmark_label = "SPY (Benchmark)"
+            spy_data = data_manager.update("etfs", ["SPY"], with_open=False)
+            if isinstance(spy_data, tuple):
+                spy_data = spy_data[0]
+            if years:
+                cutoff = spy_data.index[-1] - pd.DateOffset(years=years)
+                spy_data = spy_data[spy_data.index >= cutoff]
+            if "SPY" in spy_data.columns:
+                bench_curve = benchmark_curve(spy_data["SPY"], spy_data.index, capital)
+                start_date = spy_data.index[0].strftime("%Y-%m-%d")
+                end_date = spy_data.index[-1].strftime("%Y-%m-%d")
+
+    if not raw_strategies:
+        return None
+
+    return {
+        "market": market,
+        "strategy_runs": raw_strategies,
+        "benchmark_label": benchmark_label,
+        "benchmark_curve": bench_curve,
+        "start_date": start_date,
+        "end_date": end_date,
+        "tx_cost": tx_cost,
+        "fixed_fee": fixed_fee,
+    }
+
+
+def build_scenario_for_market(
+    raw: dict,
+    scenario: dict,
+    years: int,
+    capital: float,
+) -> dict:
+    """Materialize a scenario (with optional injections) for one market."""
+    injection_schedule = _injection_for_market(scenario.get("injection"), capital)
+    benchmark_curve = raw["benchmark_curve"]
+    benchmark_label = raw["benchmark_label"]
+
+    strategy_results = []
+    for run in raw["strategy_runs"]:
+        equity = run["equity"]
+        try:
+            result = _build_strategy_result(
+                run["name"], equity, run["trades"], injection_schedule, capital,
+            )
+            strategy_results.append(result)
+        except Exception as e:
+            log(f"    [scenario={scenario['id']}] FAILED {run['name']}: {e}")
+
+    # Re-scale benchmark for DCA scenarios so the comparison stays apples-to-apples.
+    final_bench_curve = benchmark_curve
+    if injection_schedule and benchmark_curve:
+        bench_series = _equity_series_from_curve(benchmark_curve)
+        bench_returns = bench_series.pct_change().fillna(0.0)
+        from bridge import build_injection_schedule
+        from research.permanent_yearly import equity_with_cashflows
+        contribs, _ = build_injection_schedule(bench_series, injection_schedule)
+        bench_with_inj = equity_with_cashflows(bench_returns, contribs, capital)
+        final_bench_curve = [
+            {"date": d.strftime("%Y-%m-%d"), "value": round(float(v), 2)}
+            for d, v in bench_with_inj.items()
+        ]
+
+    return {
+        "meta": {
+            "market": raw["market"],
+            "years": years,
+            "initialCapital": capital,
+            "startDate": raw["start_date"],
+            "endDate": raw["end_date"],
+            "txCost": raw["tx_cost"],
+            "fixedFee": raw["fixed_fee"],
+            "injectionSchedule": injection_schedule,
+        },
+        "strategies": strategy_results,
+        "benchmark": {
+            "name": benchmark_label,
+            "equityCurve": final_bench_curve,
+        },
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Pre-compute backtest results")
+    parser.add_argument(
+        "--output", default=None,
+        help="Output directory (default: ../stock-agent/data/precomputed)",
+    )
+    parser.add_argument(
+        "--years", type=int, default=10,
+        help="Backtest period in years (default: 10)",
+    )
+    args = parser.parse_args()
+
+    output_dir = args.output or os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), "..", "stock-agent", "data", "precomputed"
+    )
+    os.makedirs(output_dir, exist_ok=True)
+
+    us_strategies = []
+    cn_strategies = []
+    for name in STRATEGY_REGISTRY:
+        meta = STRATEGY_META.get(name, {"markets": ["us", "cn"]})
+        markets = [m for m in meta.get("markets", ["us", "cn"]) if strategy_supports_market(name, m)]
+        if "us" in markets:
+            us_strategies.append(name)
+        if "cn" in markets:
+            cn_strategies.append(name)
+
+    us_capital = MARKET_CAPITAL["us"]
+    cn_capital = MARKET_CAPITAL["cn"]
+
+    # Run each market once (cached price data → fast subsequent runs across scenarios).
+    log("=" * 60)
+    log(f"US MARKET: {len(us_strategies)} strategies @ ${us_capital:,.0f}")
+    log("=" * 60)
+    # US fee schedule (IBKR-style tiered):
+    #   fee = max(2.0, 1.88 + 0.009 * shares)
+    us_raw = run_market_raw(
+        "us", us_strategies,
+        years=args.years, capital=us_capital,
+        tx_cost=0.001, fixed_fee=2.0,
+        fee_base=1.88, fee_per_share=0.009,
+    )
+
+    log("=" * 60)
+    log(f"CN MARKET: {len(cn_strategies)} strategies @ ¥{cn_capital:,.0f}")
+    log("=" * 60)
+    cn_raw = run_market_raw(
+        "cn", cn_strategies,
+        years=args.years, capital=cn_capital,
+        tx_cost=0.001, fixed_fee=5.0,
+    )
+
+    # Materialize every scenario × market by applying its injection schedule.
+    scenarios_out: dict[str, dict] = {}
+    for scenario in SCENARIOS:
+        log(f"--- scenario: {scenario['id']} ---")
+        scenarios_out[scenario["id"]] = {
+            "id": scenario["id"],
+            "label": scenario["label"],
+            "description": scenario["description"],
+            "capital": us_capital,
+            "capitalByMarket": {"us": us_capital, "cn": cn_capital},
+            "injectionSchedule": _injection_for_market(scenario["injection"], us_capital),
+            "markets": {},
+        }
+        if us_raw:
+            scenarios_out[scenario["id"]]["markets"]["us"] = build_scenario_for_market(
+                us_raw, scenario, args.years, us_capital
+            )
+        if cn_raw:
+            scenarios_out[scenario["id"]]["markets"]["cn"] = build_scenario_for_market(
+                cn_raw, scenario, args.years, cn_capital
+            )
+
+    output = {
+        "computedAt": datetime.now().isoformat(),
+        "config": {"years": args.years, "initialCapital": MARKET_CAPITAL},
+        "scenarios": scenarios_out,
+    }
+
+    output_path = os.path.join(output_dir, "backtest_results.json")
+    with open(output_path, "w") as f:
+        json.dump(_sanitize_for_json(output), f, default=str)
+
+    file_size = os.path.getsize(output_path) / (1024 * 1024)
+    log("=" * 60)
+    log(f"DONE: Written to {output_path} ({file_size:.1f} MB)")
+    for sid, sc in scenarios_out.items():
+        for mk, mv in sc["markets"].items():
+            log(f"  [{sid}] {mk.upper()}: {len(mv['strategies'])} strategies")
+    log("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

scripts/run_daily.py

@@ -0,0 +1,116 @@
+"""Daily live-trading update for all registered strategies.
+
+Iterates STRATEGY_REGISTRY and calls trader.cmd_auto for each, sharing
+fetched price panels across strategies (we still call cmd_auto so the
+state-file format stays consistent with manual runs).
+
+Usage:
+    uv run python scripts/run_daily.py [--market us|cn|all] [--capital 10000]
+                                       [--only NAME ...] [--skip NAME ...]
+
+Designed for cron:
+    # 5pm ET weekdays, after US close
+    0 17 * * 1-5  cd /path/to/quant && uv run python scripts/run_daily.py --market us
+    # 4pm CST weekdays, after CN close
+    0 16 * * 1-5  cd /path/to/quant && uv run python scripts/run_daily.py --market cn
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import time
+import types
+from datetime import datetime
+
+# Make the project root importable when invoked as `python scripts/run_daily.py`
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+# We change into the project root so trader's relative `data/` paths resolve.
+os.chdir(PROJECT_ROOT)
+
+from trader import (  # noqa: E402
+    ETF_STRATEGY_UNIVERSES,
+    STRATEGY_REGISTRY,
+    cmd_auto,
+)
+from bridge import STRATEGY_META  # noqa: E402
+
+
+def _strategy_markets(name: str) -> list[str]:
+    """Return the markets a strategy targets (per its META declaration)."""
+    meta = STRATEGY_META.get(name, {"markets": ["us"]})
+    return list(meta.get("markets") or ["us"])
+
+
+def _make_args(market: str, name: str, capital: float) -> argparse.Namespace:
+    """Build the argparse namespace cmd_auto expects."""
+    return argparse.Namespace(
+        market=market,
+        strategy=name,
+        capital=capital,
+        tx_cost=0.001,
+        fixed_fee=0.0,         # cmd_auto falls back to MARKET_FEES + tier
+        integer_shares=True,   # match the existing live state convention
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Daily live-trading update")
+    parser.add_argument("--market", default="all",
+                        help="Which market(s): 'us', 'cn', 'all' (default: all)")
+    parser.add_argument("--capital", type=float, default=10_000,
+                        help="Initial capital for any newly-initialized strategy (default: 10000)")
+    parser.add_argument("--only", nargs="+", default=None,
+                        help="If set, only run these strategy names")
+    parser.add_argument("--skip", nargs="+", default=None,
+                        help="Strategy names to skip (e.g. broken ones)")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Print what would be run without executing")
+    args = parser.parse_args()
+
+    markets = ["us", "cn"] if args.market == "all" else [args.market]
+    only = set(args.only) if args.only else None
+    skip = set(args.skip) if args.skip else set()
+
+    plan: list[tuple[str, str]] = []
+    for name in STRATEGY_REGISTRY:
+        if only and name not in only:
+            continue
+        if name in skip:
+            continue
+        for mk in _strategy_markets(name):
+            if mk in markets:
+                plan.append((mk, name))
+
+    print(f"[run_daily] {datetime.now().isoformat(timespec='seconds')} "
+          f"will process {len(plan)} (market, strategy) pairs")
+
+    if args.dry_run:
+        for mk, name in plan:
+            print(f"  [DRY] {mk:<3} {name}")
+        return 0
+
+    failures = []
+    t0 = time.time()
+    for mk, name in plan:
+        ns = _make_args(mk, name, args.capital)
+        try:
+            cmd_auto(ns)
+        except SystemExit:
+            raise
+        except Exception as e:  # noqa: BLE001
+            failures.append((mk, name, repr(e)))
+            print(f"[run_daily] FAILED {mk} {name}: {e}", file=sys.stderr)
+
+    elapsed = time.time() - t0
+    print(f"[run_daily] done in {elapsed:.1f}s — {len(plan) - len(failures)} ok, {len(failures)} failed")
+    for mk, name, err in failures:
+        print(f"  fail {mk} {name}: {err}", file=sys.stderr)
+    return 0 if not failures else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/run_daily.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Daily live-trading update entrypoint for cron.
+#
+# Examples:
+#   0 17 * * 1-5  /path/to/quant/scripts/run_daily.sh us  >> /var/log/quant_us.log 2>&1
+#   0 16 * * 1-5  /path/to/quant/scripts/run_daily.sh cn  >> /var/log/quant_cn.log 2>&1
+#
+# The script changes into the quant project root and calls run_daily.py via uv.
+# Defaults to "all" markets when no argument is passed.
+
+set -euo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
+
+cd "$PROJECT_ROOT"
+
+MARKET="${1:-all}"
+shift || true
+
+exec uv run python scripts/run_daily.py --market "$MARKET" "$@"

scripts/start-monitor.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Long-running live-trader monitor (US + CN, all strategies).
+# Started by the quant-monitor.service systemd --user unit; replaces the
+# old "tmux new -s quant; uv run python trader.py monitor" workflow.
+set -euo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
+cd "$PROJECT_ROOT"
+
+# uv lives in ~/.local/bin, which is not on systemd's default PATH.
+export PATH="/home/gahow/.local/bin:$PATH"
+
+# Market data is fetched through the lab proxy (same as interactive `proxyOn`).
+# QUANT_MARKET_PROXY lives in the untracked repo-root .env.
+if [[ -f .env ]]; then
+  set -a; source .env; set +a
+fi
+if [[ -n "${QUANT_MARKET_PROXY:-}" ]]; then
+  export http_proxy="$QUANT_MARKET_PROXY" https_proxy="$QUANT_MARKET_PROXY" all_proxy="$QUANT_MARKET_PROXY"
+fi
+
+exec uv run python trader.py monitor