From c34367161d2088b6df6eef21c6b4d8dad2f2d5dd Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 9 Oct 2026 20:06:34 -0700 Subject: [PATCH] research: ORB (STOP), overnight-only momentum (STOP), Alpaca paper forward test - orb_stocks_in_play: 5-min bounds -70%/yr (conservative) vs +71%/yr (optimistic); the optimistic bound breaks even at 3.6 bp slippage a side. - overnight_only: the momentum sleeve's names earn their return overnight (Sharpe 1.57) and nothing intraday; held close->open only, net Sharpe 1.29-1.42 at 1 bp a side, below the 1.5 gate. - overnight_paper.py + systemd unit: MOC buy / MOO sell of the top 10 on the Alpaca paper account from 2026-10-12, with a fills-vs-prints report. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 12 +- deploy/start-overnight-paper.sh | 13 + .../quant-trading-overnight-paper.service | 14 + overnight_paper.py | 462 ++++++++++++++++++ research/lab/orb_stocks_in_play.py | 21 + research/lab/overnight_only.py | 284 +++++++++++ 6 files changed, 804 insertions(+), 2 deletions(-) create mode 100755 deploy/start-overnight-paper.sh create mode 100644 deploy/systemd/quant-trading-overnight-paper.service create mode 100644 overnight_paper.py create mode 100644 research/lab/overnight_only.py diff --git a/CLAUDE.md b/CLAUDE.md index ce939d3..db4541c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,6 +43,12 @@ uv run python agent_review.py candidates # after the US close: top-50 pack uv run python agent_review.py submit data/agent/us/draft_.json # before 16:00 New York of the next session uv run python agent_review.py evaluate # score IC, veto outcomes, book vs overnight_recovery_top10 +# Overnight-only momentum book on Alpaca PAPER (MOC buy top 10 at close-20 min, MOO sell at open-20 min; refuses non-paper endpoints) +uv run python overnight_paper.py signal # next session's top 10, no orders +uv run python overnight_paper.py buy|sell --dry-run # show the orders it would send +uv run python overnight_paper.py report # paper fills vs official open/close prints, nightly returns +uv run python overnight_paper.py daemon # the scheduler (service quant-trading-overnight-paper) + # Precompute the strategy page's backtests (writes data/precomputed/) uv run python precompute_backtest.py @@ -90,7 +96,8 @@ PYTHONPATH=. uv run python -m research.lab.turn_of_month # turn-of-mo PYTHONPATH=. uv run python -m research.lab.short_vol [--no-proxy] # short VIX futures with a VIX/VIX3M filter, 2006-26 (quick check; STOP 2026-10-08) PYTHONPATH=. uv run python -m research.lab.fetch_alpaca etfs|stocks|status # Alpaca free SIP intraday bars 2016+ (1-min ETFs, 5-min liquid S&P 1500; keys from .env, never logged) PYTHONPATH=. uv run python -m research.lab.intraday_momentum # SPY noise-area intraday momentum replication + 5 ETFs (preregistered; STOP 2026-10-09) -PYTHONPATH=. uv run python -m research.lab.orb_stocks_in_play # 5-min opening-range breakout on the top-20 stocks in play (preregistered) +PYTHONPATH=. uv run python -m research.lab.orb_stocks_in_play # 5-min opening-range breakout on the top-20 stocks in play (preregistered; STOP 2026-10-09: 5-min bounds -70%/yr vs +71%/yr, optimistic breakeven slippage 3.6 bp/side) +PYTHONPATH=. uv run python -m research.lab.overnight_only # momentum names held close->open only, MOC/MOO costs (preregistered; STOP 2026-10-09: net Sharpe 1.29-1.42 < 1.5, but the sleeve's whole return is overnight) uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs) uv run python -m research.lab.fetch_us_fundamentals --universe small # the same for S&P 600 (IJR) PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] [--universe small] # S&P 400 sleeves vs IJH; S&P 600 replication vs IJR (all REJECT 2026-10-08) @@ -190,7 +197,7 @@ The comparison writes its fixed budget and acceptance rule before running and re ## Server Deployment -Three `systemd --user` services, defined in `deploy/systemd/`, run from `/home/gahow/projects/quant-trading`: +Four `systemd --user` services, defined in `deploy/systemd/`, run from `/home/gahow/projects/quant-trading`: | Service | Script | Role | |---|---|---| @@ -200,6 +207,7 @@ Three `systemd --user` services, defined in `deploy/systemd/`, run from `/home/g ```bash cp deploy/systemd/*.service ~/.config/systemd/user/ +| `quant-trading-overnight-paper` | `deploy/start-overnight-paper.sh` | `overnight_paper.py daemon`: overnight-only momentum on the Alpaca paper account (from 2026-10-12) | systemctl --user daemon-reload systemctl --user enable --now quant-trading-monitor quant-trading-api quant-trading-vite journalctl --user -u quant-trading-monitor -f diff --git a/deploy/start-overnight-paper.sh b/deploy/start-overnight-paper.sh new file mode 100755 index 0000000..7794c35 --- /dev/null +++ b/deploy/start-overnight-paper.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Overnight-only momentum book on Alpaca's PAPER account (overnight_paper.py daemon). +# Started by the quant-trading-overnight-paper.service systemd --user unit. +# Keys are read from the repo-root .env by the script itself; Alpaca is reached directly (no proxy). +set -euo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" +cd "$PROJECT_ROOT" + +export PATH="/home/gahow/.local/bin:$PATH" + +exec uv run python overnight_paper.py daemon diff --git a/deploy/systemd/quant-trading-overnight-paper.service b/deploy/systemd/quant-trading-overnight-paper.service new file mode 100644 index 0000000..6277850 --- /dev/null +++ b/deploy/systemd/quant-trading-overnight-paper.service @@ -0,0 +1,14 @@ +[Unit] +Description=Overnight-only momentum book on Alpaca paper (MOC buy, MOO sell) +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=/home/gahow/projects/quant-trading +ExecStart=/home/gahow/projects/quant-trading/deploy/start-overnight-paper.sh +Restart=always +RestartSec=30 +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=default.target diff --git a/overnight_paper.py b/overnight_paper.py new file mode 100644 index 0000000..710a7e0 --- /dev/null +++ b/overnight_paper.py @@ -0,0 +1,462 @@ +""" +Forward test of the overnight-only momentum book on Alpaca's PAPER account +(research: research/lab/overnight_only.py). Each session it buys the top 10 of +overnight_recovery's score with market-on-close orders and sells every +position with market-on-open orders at the next session, so the book is flat +during the day. The test measures what the backtest cannot: fills against the +official close/open prints, and the forward record. + + uv run python overnight_paper.py signal # today's list, no orders + uv run python overnight_paper.py buy [--dry-run] # MOC orders; between close-25 and close-11 min + uv run python overnight_paper.py sell [--dry-run] # MOO sells; between 19:00 ET and open-2 min + uv run python overnight_paper.py report # fills vs official prints, nightly returns + uv run python overnight_paper.py daemon # runs sell / buy / report on Alpaca's calendar + +Rules (as in the study's sleeve_score candidate): + score strategies.robust_momentum.signal_overnight_recovery (12-1 month + overnight momentum + distance from the 126-day low, percentile + blend) on Alpaca SIP daily bars (split/dividend adjusted) through + the previous session, current S&P 500 members + book top 10, equal weight, whole shares (Alpaca accepts only whole-share + MOC/MOO orders), 1x the account equity, no leverage + orders buy: market, time_in_force "cls", sent at close - 20 min (Alpaca + rejects MOC orders after 15:50 ET); sell: market, "opg", sent at + open - 20 min. A position left over from a failed sell is resized + to the new target at the close. +Keys come from ALPACA_API_ENDPOINT / ALPACA_API_KEY / ALPACA_API_SECRET in the +environment or .env; they are never printed. The script refuses any endpoint +that is not paper-api.alpaca.markets. + +Boundary: Alpaca's paper engine simulates fills from live quotes; it does not +reproduce the exchange auctions or market impact. The report therefore scores +both the paper fills and the official open/close prints of the same names. + +State and logs: data/overnight_paper/ (orders.jsonl, signals/.json, +state.json, fills.csv, nights.csv). +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +import time +from datetime import date, datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +import pandas as pd +import requests + +from strategies.robust_momentum import signal_overnight_recovery + +OUT = os.path.join("data", "overnight_paper") +DATA_URL = "https://data.alpaca.markets/v2" +PAPER_HOST = "paper-api.alpaca.markets" +NY = ZoneInfo("America/New_York") +TOP_N = 10 +HISTORY_DAYS = 450 # calendar days of daily bars (score needs ~280 sessions) +BUY_BEFORE_CLOSE = timedelta(minutes=20) +BUY_CUTOFF = timedelta(minutes=11) # Alpaca: MOC orders close at 15:50 ET +SELL_BEFORE_OPEN = timedelta(minutes=20) +SELL_CUTOFF = timedelta(minutes=2) # Alpaca: MOO orders close at 09:28 ET +REPORT_AFTER_OPEN = timedelta(minutes=30) + + +# --------------------------------------------------------------------------- +# Alpaca client (paper only; keys never printed) +# --------------------------------------------------------------------------- + +def _env() -> dict[str, str]: + env = dict(os.environ) + if os.path.exists(".env"): + for line in open(".env"): + if "=" in line and not line.lstrip().startswith("#"): + k, v = line.split("=", 1) + env.setdefault(k.strip(), v.strip().strip("'\"")) + return env + + +class Alpaca: + def __init__(self) -> None: + env = _env() + base = env["ALPACA_API_ENDPOINT"].rstrip("/") + if PAPER_HOST not in base: + raise SystemExit("ALPACA_API_ENDPOINT is not the paper API; refusing to trade") + self.base = base if base.endswith("/v2") else base + "/v2" + self.s = requests.Session() + self.s.headers.update({"APCA-API-KEY-ID": env["ALPACA_API_KEY"], + "APCA-API-SECRET-KEY": env["ALPACA_API_SECRET"]}) + + def _req(self, method: str, url: str, **kw) -> requests.Response: + for attempt in range(6): + try: + r = self.s.request(method, url, timeout=30, **kw) + except requests.RequestException: + time.sleep(2 ** attempt) + continue + if r.status_code == 429 or r.status_code >= 500: + time.sleep(2 ** attempt) + continue + return r + raise RuntimeError(f"{method} {url.split('/v2')[-1]}: retries exhausted") + + def get(self, path: str, data: bool = False, **params): + r = self._req("GET", (DATA_URL if data else self.base) + path, params=params) + if r.status_code != 200: + raise RuntimeError(f"GET {path}: HTTP {r.status_code} {r.text[:200]}") + return r.json() + + def order(self, payload: dict) -> dict: + r = self._req("POST", self.base + "/orders", json=payload) + body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {"text": r.text} + return {"http": r.status_code, **body} + + def cancel(self, order_id: str) -> int: + return self._req("DELETE", f"{self.base}/orders/{order_id}").status_code + + +def alpaca_symbol(yahoo: str) -> str: + return yahoo.replace("-", ".") + + +# --------------------------------------------------------------------------- +# calendar +# --------------------------------------------------------------------------- + +def sessions(cl: Alpaca, start: date, end: date) -> pd.DataFrame: + rows = cl.get("/calendar", start=start.isoformat(), end=end.isoformat()) + out = [] + for c in rows: + d = date.fromisoformat(c["date"]) + o = datetime.combine(d, datetime.strptime(c["open"], "%H:%M").time(), NY) + k = datetime.combine(d, datetime.strptime(c["close"], "%H:%M").time(), NY) + out.append({"date": d, "open": o, "close": k}) + return pd.DataFrame(out).set_index("date") + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +# --------------------------------------------------------------------------- +# signal +# --------------------------------------------------------------------------- + +def sp500() -> list[str]: + try: + from universe import get_sp500 + return get_sp500() + except Exception as exc: # noqa: BLE001 - Wikipedia unreachable: last cached list + with open(os.path.join("data", "universe_sp500.json")) as fh: + cached = json.load(fh) + print(f"S&P 500 list from cache dated {cached.get('date')} ({type(exc).__name__})", flush=True) + return cached["tickers"] + + +def _data_end(end: date) -> str: + """The free plan refuses SIP data from the last 15 minutes; a bare date counts as that whole day.""" + stop = min(datetime.combine(end, datetime.strptime("23:59", "%H:%M").time(), NY), # daily bars are stamped 00:00 ET + now_utc() - timedelta(minutes=20)) + return stop.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def daily_bars(cl: Alpaca, symbols: list[str], start: date, end: date) -> dict[str, pd.DataFrame]: + """Adjusted SIP daily open/close, one column per Yahoo-spelled symbol.""" + rows = [] + amap = {alpaca_symbol(s): s for s in symbols} + names = list(amap) + for i in range(0, len(names), 100): + params = {"symbols": ",".join(names[i:i + 100]), "timeframe": "1Day", "start": start.isoformat(), + "end": _data_end(end), "adjustment": "all", "feed": "sip", "limit": 10000} + while True: + j = cl.get("/stocks/bars", data=True, **params) + for sym, bars in (j.get("bars") or {}).items(): + for b in bars: + rows.append((amap[sym], b["t"][:10], b["o"], b["c"])) + tok = j.get("next_page_token") + if not tok: + break + params["page_token"] = tok + df = pd.DataFrame(rows, columns=["sym", "date", "open", "close"]) + df["date"] = pd.to_datetime(df["date"]) + return {f: df.pivot(index="date", columns="sym", values=f).sort_index() for f in ("open", "close")} + + +def compute_signal(cl: Alpaca, session: date) -> dict: + """Top 10 for the close of ``session`` from bars through the previous session.""" + cal = sessions(cl, session - timedelta(days=10), session) + prev = [d for d in cal.index if d < session] + if not prev: + raise RuntimeError(f"no session before {session}") + last = prev[-1] + syms = sp500() + bars = daily_bars(cl, syms, session - timedelta(days=HISTORY_DAYS), last) + if bars["close"].index[-1].date() != last: + raise RuntimeError(f"daily bars end {bars['close'].index[-1].date()}, expected {last}") + score = signal_overnight_recovery({"open": bars["open"], "close": bars["close"]}) + row = score.iloc[-1].dropna().sort_values(ascending=False, kind="mergesort") + top = list(row.index[:TOP_N]) + return {"session": session.isoformat(), "bars_through": last.isoformat(), "universe": len(syms), + "scored": int(row.size), "top": top, "scores": {s: round(float(row[s]), 4) for s in top}, + "prev_close": {s: float(bars["close"][s].iloc[-1]) for s in top}} + + +# --------------------------------------------------------------------------- +# state / logs +# --------------------------------------------------------------------------- + +def _state() -> dict: + path = os.path.join(OUT, "state.json") + return json.load(open(path)) if os.path.exists(path) else {} + + +def _save_state(st: dict) -> None: + os.makedirs(OUT, exist_ok=True) + tmp = os.path.join(OUT, ".state.json.tmp") + json.dump(st, open(tmp, "w"), indent=1) + os.replace(tmp, os.path.join(OUT, "state.json")) + + +def _log_order(kind: str, payload: dict, resp: dict) -> None: + os.makedirs(OUT, exist_ok=True) + keep = {k: resp.get(k) for k in ("http", "id", "client_order_id", "status", "symbol", "qty", "side", + "time_in_force", "submitted_at", "message", "text")} + with open(os.path.join(OUT, "orders.jsonl"), "a") as fh: + fh.write(json.dumps({"logged_at": now_utc().isoformat(), "kind": kind, "payload": payload, + "response": keep}) + "\n") + + +def _submit(cl: Alpaca, kind: str, payload: dict, dry: bool) -> None: + tag = f"{payload['side']:4s} {payload['qty']:>6s} {payload['symbol']:6s} {payload['time_in_force']}" + if dry: + print(f" [dry-run] {tag}", flush=True) + return + resp = cl.order(payload) + _log_order(kind, payload, resp) + ok = resp["http"] in (200, 201) + dup = resp["http"] == 422 and "client_order_id" in json.dumps(resp).lower() + print(f" {tag} -> {'ok ' + str(resp.get('status')) if ok else 'already sent' if dup else 'HTTP ' + str(resp['http']) + ' ' + str(resp.get('message', resp.get('text', '')))[:120]}", + flush=True) + + +# --------------------------------------------------------------------------- +# actions +# --------------------------------------------------------------------------- + +def positions(cl: Alpaca) -> dict[str, int]: + return {p["symbol"]: int(float(p["qty"])) for p in cl.get("/positions")} + + +def open_orders(cl: Alpaca) -> list[dict]: + return cl.get("/orders", status="open", limit=500) + + +def latest_prices(cl: Alpaca, symbols: list[str]) -> dict[str, float]: + """Last trade on the free real-time feed (IEX); sizing only.""" + j = cl.get("/stocks/trades/latest", data=True, symbols=",".join(alpaca_symbol(s) for s in symbols), feed="iex") + return {s: float(j["trades"][alpaca_symbol(s)]["p"]) for s in symbols if alpaca_symbol(s) in j.get("trades", {})} + + +def buy(cl: Alpaca, dry: bool = False, force: bool = False) -> None: + today = datetime.now(NY).date() + cal = sessions(cl, today, today) + if today not in cal.index: + print(f"{today}: no session", flush=True) + return + close = cal.loc[today, "close"] + if not force and not (close - BUY_BEFORE_CLOSE - timedelta(minutes=5) <= now_utc() <= close - BUY_CUTOFF): + print(f"{today}: outside the MOC window (close {close:%H:%M} ET); use --force for a dry check", flush=True) + if not dry: + return + st = _state() + if st.get("bought") == today.isoformat() and not dry: + print(f"{today}: MOC orders already sent", flush=True) + return + sig = compute_signal(cl, today) + os.makedirs(os.path.join(OUT, "signals"), exist_ok=True) + json.dump(sig, open(os.path.join(OUT, "signals", f"{today}.json"), "w"), indent=1) + acct = cl.get("/account") + equity = float(acct["equity"]) + px = latest_prices(cl, sig["top"]) + held = positions(cl) + pending = {o["symbol"] for o in open_orders(cl) if o["time_in_force"] == "cls"} + target = {} + for s in sig["top"]: + p = px.get(s) or sig["prev_close"][s] + target[alpaca_symbol(s)] = math.floor(equity / TOP_N / p) + print(f"{today}: equity {equity:,.0f}, bars through {sig['bars_through']}, top {sig['top']}", flush=True) + for sym in sorted(set(target) | set(held)): + if sym in pending: + print(f" {sym}: MOC order already open", flush=True) + continue + delta = target.get(sym, 0) - held.get(sym, 0) + if delta == 0: + continue + payload = {"symbol": sym, "qty": str(abs(delta)), "side": "buy" if delta > 0 else "sell", + "type": "market", "time_in_force": "cls", "client_order_id": f"onb-{today}-{sym}"} + _submit(cl, "moc", payload, dry) + if not dry: + st["bought"] = today.isoformat() + _save_state(st) + + +def sell(cl: Alpaca, dry: bool = False) -> None: + now = now_utc() + cal = sessions(cl, (now - timedelta(days=1)).date(), (now + timedelta(days=10)).date()) + nxt = cal[cal["open"] > now] + if nxt.empty: + print("no upcoming session", flush=True) + return + session, opn = nxt.index[0], nxt["open"].iloc[0] + if not dry and now > opn - SELL_CUTOFF: + print(f"{session}: too late for MOO orders", flush=True) + return + st = _state() + if st.get("sold") == session.isoformat() and not dry: + print(f"{session}: MOO orders already sent", flush=True) + return + held = {s: q for s, q in positions(cl).items() if q > 0} + pending = {o["symbol"] for o in open_orders(cl) if o["time_in_force"] == "opg" and o["side"] == "sell"} + print(f"{session}: MOO sells for {len(held)} positions", flush=True) + for sym, q in sorted(held.items()): + if sym in pending: + print(f" {sym}: MOO order already open", flush=True) + continue + payload = {"symbol": sym, "qty": str(q), "side": "sell", "type": "market", "time_in_force": "opg", + "client_order_id": f"ons-{session}-{sym}"} + _submit(cl, "moo", payload, dry) + if not dry: + st["sold"] = session.isoformat() + _save_state(st) + + +def report(cl: Alpaca) -> None: + """Fills against the official prints (Alpaca SIP daily open/close, unadjusted) and nightly returns.""" + orders, after = [], (now_utc() - timedelta(days=400)).isoformat() + while True: + page = cl.get("/orders", status="closed", limit=500, direction="asc", after=after) + orders += page + if len(page) < 500: + break + after = page[-1]["submitted_at"] + rows = [] + for o in orders: + cid = o.get("client_order_id", "") + if not cid.startswith(("onb-", "ons-")) or o.get("filled_avg_price") is None: + continue + rows.append({"session": cid[4:14], "symbol": o["symbol"], "side": o["side"], "tif": o["time_in_force"], + "qty": float(o["filled_qty"]), "fill": float(o["filled_avg_price"]), + "filled_at": o["filled_at"]}) + if not rows: + print("no fills yet", flush=True) + return + F = pd.DataFrame(rows) + d0 = date.fromisoformat(F["session"].min()) + d1 = date.fromisoformat(F["session"].max()) + rows = [] + syms = sorted(F["symbol"].unique()) + for i in range(0, len(syms), 100): + params = {"symbols": ",".join(syms[i:i + 100]), "timeframe": "1Day", "start": d0.isoformat(), + "end": _data_end(d1), "adjustment": "raw", "feed": "sip", "limit": 10000} + while True: + j = cl.get("/stocks/bars", data=True, **params) + for sym, bars in (j.get("bars") or {}).items(): + rows += [(sym, b["t"][:10], b["o"], b["c"]) for b in bars] + if not j.get("next_page_token"): + break + params["page_token"] = j["next_page_token"] + P = pd.DataFrame(rows, columns=["symbol", "session", "open", "close"]) + F = F.merge(P, on=["symbol", "session"], how="left") + F["ref"] = F["close"].where(F["tif"] == "cls", F["open"]) + # cost in bp: positive = worse than the official print + sign = F["side"].map({"buy": 1, "sell": -1}) + F["slip_bp"] = sign * (F["fill"] / F["ref"] - 1) * 1e4 + F.to_csv(os.path.join(OUT, "fills.csv"), index=False) + b = F[F["tif"] == "cls"] + s = F[F["tif"] == "opg"] + print(f"fills {len(F)}: MOC slip {b['slip_bp'].mean():.2f} bp (median {b['slip_bp'].median():.2f}), " + f"MOO slip {s['slip_bp'].mean():.2f} bp (median {s['slip_bp'].median():.2f})", flush=True) + # nights: buy session d (close) -> sell next session (open) + nights = [] + for d, g in b.groupby("session"): + later = s[s["session"] > d] + if later.empty: + continue + nd = later["session"].min() + sg = later[later["session"] == nd].set_index("symbol") + bg = g.set_index("symbol") + both = bg.index.intersection(sg.index) + if both.empty: + continue + cost = (bg.loc[both, "qty"] * bg.loc[both, "fill"]).sum() + pnl = (sg.loc[both, "qty"] * sg.loc[both, "fill"] - bg.loc[both, "qty"] * bg.loc[both, "fill"]).sum() + model = (sg.loc[both, "open"] / bg.loc[both, "close"] - 1).mean() + nights.append({"buy_session": d, "sell_session": nd, "names": len(both), + "paper_ret": pnl / cost, "print_ret": model}) + if nights: + N = pd.DataFrame(nights) + N.to_csv(os.path.join(OUT, "nights.csv"), index=False) + print(N.tail(10).to_string(index=False, float_format=lambda x: f"{x:.4f}"), flush=True) + print(f"nights {len(N)}: paper mean {N['paper_ret'].mean()*1e4:.1f} bp/night, " + f"prints mean {N['print_ret'].mean()*1e4:.1f} bp/night", flush=True) + + +# --------------------------------------------------------------------------- +# daemon +# --------------------------------------------------------------------------- + +def daemon() -> None: + cl = Alpaca() + print(f"{now_utc():%Y-%m-%d %H:%M} UTC daemon started (paper account)", flush=True) + done: set[tuple[str, str]] = set() + while True: + try: + now = now_utc() + cal = sessions(cl, (now - timedelta(days=1)).date(), (now + timedelta(days=10)).date()) + for d, row in cal.iterrows(): + key = d.isoformat() + sell_at, buy_at = row["open"] - SELL_BEFORE_OPEN, row["close"] - BUY_BEFORE_CLOSE + if ("sell", key) not in done and sell_at <= now < row["open"] - SELL_CUTOFF: + sell(cl) + done.add(("sell", key)) + if ("buy", key) not in done and buy_at <= now < row["close"] - BUY_CUTOFF: + buy(cl) + done.add(("buy", key)) + if ("report", key) not in done and row["open"] + REPORT_AFTER_OPEN <= now < row["close"]: + report(cl) + done.add(("report", key)) + except Exception as exc: # noqa: BLE001 - keep the loop alive; the next tick retries + print(f"{now_utc():%Y-%m-%d %H:%M} UTC error: {type(exc).__name__}: {str(exc)[:300]}", flush=True) + time.sleep(60) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("what", choices=["signal", "buy", "sell", "report", "daemon"]) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--force", action="store_true", help="buy: ignore the time window (still honours --dry-run)") + a = ap.parse_args() + if a.what == "daemon": + daemon() + return + cl = Alpaca() + if a.what == "signal": + today = datetime.now(NY).date() + cal = sessions(cl, today, today + timedelta(days=10)) + nxt = cal[cal["close"] > now_utc()].index[0] + sig = compute_signal(cl, nxt) + print(json.dumps(sig, indent=1)) + elif a.what == "buy": + if a.force and not a.dry_run: + sys.exit("--force is only for --dry-run checks") + buy(cl, dry=a.dry_run, force=a.force) + elif a.what == "sell": + sell(cl, dry=a.dry_run) + else: + report(cl) + + +if __name__ == "__main__": + main() diff --git a/research/lab/orb_stocks_in_play.py b/research/lab/orb_stocks_in_play.py index 4bf0309..a318673 100644 --- a/research/lab/orb_stocks_in_play.py +++ b/research/lab/orb_stocks_in_play.py @@ -31,6 +31,27 @@ Deviations, fixed before the run: The rule is written to prereg.json before the run; the script refuses to overwrite results.json. 2016-2023 is the paper's own sample; 2024-01 onward is after it. + +The fetch was pre-filtered to names that passed the filters on at least one +member day (fetch_alpaca.universe, liquid=True). The daily filters above +still apply. + +Result (2026-10-09): STOP. The conservative bound fails all gates; the +optimistic bound fails O3 (stress). + - Data: 1,686 files, 356,908 eligible name-days, 42,807 trades. + - Conservative (a stop inside the entry bar counts as hit): -70.5%/yr, + Sharpe -7.5, win rate 8%. + - Optimistic (no stop inside the entry bar): + - Commission only: 71.5%/yr, Sharpe 2.14, MaxDD -29%. Paper-span + Sharpe 2.19, 2024+ 1.98, halves 2.32 / 1.97. + - +5 bp per side: Sharpe -0.86. + - A 10%-ATR stop sits inside a typical 5-minute range, so bar resolution + decides the sign. + - Even the optimistic bound breaks even at only 3.6 bp of slippage per + side on top of commission (1 bp: Sharpe 1.54; 2 bp: 0.94; 3 bp: 0.34), + at 3.2x average gross leverage. + - Stop-order entries on breakouts of stocks in play rarely fill within + a few bp, so 1-minute bars (RESOLVE) would not change the decision. """ from __future__ import annotations diff --git a/research/lab/overnight_only.py b/research/lab/overnight_only.py new file mode 100644 index 0000000..dec578d --- /dev/null +++ b/research/lab/overnight_only.py @@ -0,0 +1,284 @@ +""" +Hold the momentum sleeve's names only overnight (buy at the close, sell at the +next open) and stay flat during the day. Intraday trading is allowed since +2026-10-09, so an overnight-only book is tradable with market-on-close and +market-on-open orders, and it leaves the capital free for an intraday strategy +during the session. + + PYTHONPATH=. uv run python -m research.lab.overnight_only + +Background: Lou, Polk & Skouras, "A tug of war: Overnight versus intraday +expected returns" (JFE 2019) find that momentum profits accrue overnight and +partly reverse intraday. overnight_recovery_top10 ranks on 12-1 month +overnight returns, so its own return may sit mostly in the overnight leg. + +Rules (one book per candidate, equal weight, rebuilt every night): + night t from the close of t-1 to the open of t + selection top 10 by the candidate score read at the close of t-2 (one day + of slack so the MOC order can be sent before the close of t-1), + PIT S&P 500 members on t-2 + return open_t / close_{t-1} - 1 per name (Yahoo adjusted bars, so + dividends and splits net out); a missing open falls back to + close_t + costs every night trades the whole book twice: c per side, baseline + c = 1 bp (auction fills, $0 commission, SEC fee ~0.3 bp on + sells), stress c = 3 bp + cash earns nothing; Sharpe = mean / sd of the book's own returns + +Candidates: + sleeve_score overnight_mom (12-1) + rec_126 percentile blend, the live + sleeve's score + overnight_12_1 overnight_mom alone + overnight_1m last month's overnight return (LPS: overnight returns + persist at short horizons) + +Reported only: the overnight / intraday split of the sleeve's names held for +10 days (the live cadence, no hysteresis), and the equal-weight +member universe held overnight only. + +Open-price fidelity (N5): Yahoo opens against the 09:30 bar of the Alpaca 5-min +SIP files on disk at run time (2016+), on all name-days and on the name-days +the candidates select. If N5 fails the verdict is DATA. + +Every period here was examined before (the sleeve's score was chosen on this +panel). 2019+ is after the LPS publication, not independent OOS. + +The rule is written to prereg.json before the run; the script refuses to +overwrite results.json. + +Result (2026-10-09): STOP. No candidate reaches N1 (full-span net Sharpe 1.5). +N5 passes: on 288 names, 2016+, Yahoo and Alpaca opens give overnight returns +with corr 0.96 on all name-days and 0.996 on selected ones; the mean +difference is 0.03-0.3 bp. + + candidate (net, 1 bp) CAGR Sharpe halves 2019+ 3 bp MaxDD + sleeve_score 30.4% 1.29 1.34 / 1.24 1.17 0.84 -31% + overnight_12_1 38.0% 1.42 1.59 / 1.24 1.16 1.02 -31% + overnight_1m 25.9% 1.08 1.47 / 0.53 0.54 0.66 -33% + +Reported only: + - The live sleeve's names held 10 days (gross) earn the whole return + overnight: overnight leg 13.7 bp/day, Sharpe 1.57 (halves 1.62 / 1.50, + 2019+ 1.43); intraday leg -0.8 bp/day, Sharpe -0.07; close to close + Sharpe 0.87. + - Nightly re-selection adds nothing over the 10-day list (13.5 vs 13.7 + bp/day). The equal-weight universe earns 3.5 bp/day overnight (Sharpe 0.73). + - Costs decide the result: 2 sides a night at 1 bp take 2 bp of the 13.5 + bp/day gross. Breakeven is about 6.7 bp a side. +""" + +from __future__ import annotations + +import json +import os + +import numpy as np +import pandas as pd + +from research.lab.factor_scan import RESULTS, load +from research.lab.factors import overnight_mom, overnight_mom_1, rec_126 +from research.lab.us_sweep import _blend + +OUT = os.path.join(RESULTS, "overnight_only") +BARS = os.path.join("data", "research", "intraday", "alpaca", "5Min") +SPAN = ("2006-10-02", "2026-09-24") +HALVES = [("2006-10-02", "2016-12-30"), ("2017-01-03", "2026-09-24")] +POST_PUB = ("2019-01-01", "2026-09-24") +TOP_N, HOLD_DAYS = 10, 10 +COST_BASE, COST_STRESS = 0.0001, 0.0003 + +PREREG = { + "question": "Does holding the momentum sleeve's names only overnight (MOC buy, MOO sell) earn a net " + "Sharpe high enough to be a stand-alone stream that frees intraday capital?", + "rules": "top 10 equal weight by score at close t-2, held close t-1 -> open t, rebuilt nightly; " + "costs 2 x c per night on the whole book, c = 1 bp base / 3 bp stress; no cash interest", + "candidates": ["sleeve_score", "overnight_12_1", "overnight_1m"], + "span": SPAN, "halves": HALVES, "post_publication": POST_PUB, + "gates": {"N1": "full-span net Sharpe (1 bp) >= 1.5", + "N2": "net Sharpe (1 bp) >= 1.2 in both halves", + "N3": "net Sharpe (1 bp) >= 1.0 in 2019-01 .. 2026-09", + "N4": "full-span net Sharpe at 3 bp/side >= 1.0", + "N5": "open fidelity vs Alpaca 09:30 bar (2016+): corr of overnight returns >= 0.9 on all " + "name-days and |mean diff| <= 3 bp on selected name-days"}, + "decision": {"CONTINUE": "a candidate passes N1-N4 and N5 passes -> check MOC/MOO auction fills on Alpaca " + "intraday data and the combination with an intraday stream on the same capital", + "DATA": "N5 fails -> rebuild overnight returns from Alpaca opens before any verdict", + "STOP": "otherwise"}, + "multiplicity": "3 candidates; a lone pass by one is weaker evidence than the sleeve score passing", + "data_status": "all periods previously examined; 2019+ is post-publication, not independent OOS", +} + + +def describe(r: pd.Series) -> dict: + r = r.dropna() + eq = (1 + r).cumprod() + yrs = len(r) / 252 + return {"cagr": float(eq.iloc[-1] ** (1 / yrs) - 1), "vol": float(r.std() * np.sqrt(252)), + "sharpe": float(r.mean() / r.std() * np.sqrt(252)), "mdd": float((eq / eq.cummax() - 1).min()), + "mean_bp_per_day": float(r.mean() * 1e4)} + + +def legs(p) -> tuple[pd.DataFrame, pd.DataFrame]: + """Per name-day overnight (open_t / close_{t-1}) and intraday (close_t / open_t) returns.""" + c = p.close.ffill() + o = p.open.where(p.open.notna(), p.close) + return o / c.shift(1) - 1, p.close / o - 1 + + +def top_mask(score: pd.DataFrame, n: int) -> pd.DataFrame: + rk = score.rank(axis=1, ascending=False, method="first") + return rk <= n + + +def nightly(sel: pd.DataFrame, on: pd.DataFrame) -> pd.Series: + """Equal-weight overnight return of the selected names (selection already aligned to night t).""" + x = on.where(sel) + return x.mean(axis=1).where(sel.sum(axis=1) > 0) + + +def held_split(score: pd.DataFrame, on: pd.DataFrame, intra: pd.DataFrame) -> dict: + """Sleeve names held 10 days (refresh at close t-1 with score through t-2), daily reset to equal + weight, averaged over the 10 phases: overnight leg, intraday leg, close-to-close.""" + rows = {"overnight": [], "intraday": [], "close_to_close": []} + sel_daily = top_mask(score.shift(2), TOP_N) + idx = np.arange(len(score)) + for off in range(HOLD_DAYS): + sel = sel_daily.astype(float) + sel.loc[(idx % HOLD_DAYS) != off] = np.nan + sel = sel.ffill().fillna(0.0).astype(bool) + o = on.where(sel).mean(axis=1) + i = intra.where(sel).mean(axis=1) + cc = ((1 + on) * (1 + intra) - 1).where(sel).mean(axis=1) + rows["overnight"].append(o); rows["intraday"].append(i); rows["close_to_close"].append(cc) + return {k: pd.concat(v, axis=1).mean(axis=1) for k, v in rows.items()} + + +def alpaca_opens(columns) -> tuple[pd.DataFrame, pd.DataFrame, int]: + """Daily 09:30 bar open and last-bar close per symbol from the 5-min files on disk.""" + opens, closes = {}, {} + files = [f[:-4] for f in os.listdir(BARS) if f.endswith(".pkl")] if os.path.isdir(BARS) else [] + for s in files: + if s not in columns: + continue + df = pd.read_pickle(os.path.join(BARS, f"{s}.pkl")) + day = df.index.tz_localize(None).normalize() + first = df.groupby(day).head(1) + first = first[(first.index.hour == 9) & (first.index.minute == 30)] + opens[s] = pd.Series(first["o"].values.astype(float), index=first.index.tz_localize(None).normalize()) + closes[s] = df["c"].groupby(day).last().astype(float) + return pd.DataFrame(opens), pd.DataFrame(closes), len(opens) + + +def fidelity(p, sels: dict[str, pd.DataFrame]) -> dict: + ao, ac, n = alpaca_opens(set(p.close.columns)) + if n == 0: + return {"symbols": 0} + cols = list(ao.columns) + dates = p.dates[p.dates >= "2016-01-04"] + ao, ac = ao.reindex(index=dates, columns=cols), ac.reindex(index=dates, columns=cols) + # overnight with each source's own prior close (adjustment factors cancel within a source) + on_y = (p.open / p.close.ffill().shift(1) - 1).reindex(index=dates, columns=cols) + on_a = ao / ac.shift(1) - 1 + mem = p.member.reindex(index=dates, columns=cols).fillna(False) + ok = mem & on_y.notna() & on_a.notna() + y, a = on_y.where(ok), on_a.where(ok) + out = {"symbols": n, "name_days": int(ok.sum().sum()), + "corr_all": float(np.corrcoef(y.stack().values, a.stack().values)[0, 1]), + "mean_diff_all_bp": float((y - a).stack().mean() * 1e4), + "median_abs_diff_all_bp": float((y - a).abs().stack().median() * 1e4)} + for name, sel in sels.items(): + s = sel.reindex(index=dates, columns=cols).fillna(False).astype(bool) & ok + d = (y - a).where(s).stack() + out[f"{name}_name_days"] = int(s.sum().sum()) + out[f"{name}_mean_diff_bp"] = float(d.mean() * 1e4) if len(d) else float("nan") + out[f"{name}_corr"] = (float(np.corrcoef(y.where(s).stack().values, a.where(s).stack().values)[0, 1]) + if len(d) > 10 else float("nan")) + return out + + +def main() -> None: + os.makedirs(OUT, exist_ok=True) + res_path = os.path.join(OUT, "results.json") + if os.path.exists(res_path): + raise SystemExit(f"{res_path} exists; this study does not overwrite results") + with open(os.path.join(OUT, "prereg.json"), "w") as fh: + json.dump(PREREG, fh, indent=2) + + p, _ = load("us") + mem = p.member + on, intra = legs(p) + scores = { + "sleeve_score": _blend(*[f.where(mem).rank(axis=1, pct=True) + for f in (overnight_mom(p), rec_126(p))]).where(mem), + "overnight_12_1": overnight_mom(p).where(mem), + "overnight_1m": overnight_mom_1(p).where(mem), + } + a, b = SPAN + res, daily, sels = {"candidates": {}}, {}, {} + for name, sc in scores.items(): + sel = top_mask(sc.shift(2), TOP_N) + sels[name] = sel + gross = nightly(sel, on).loc[a:b] + net, stress = gross - 2 * COST_BASE, gross - 2 * COST_STRESS + clipped = nightly(sel, on.clip(-0.2, 0.2)).loc[a:b] - 2 * COST_BASE + big = on.where(sel & (on.abs() > 0.10)).sum(axis=1).loc[a:b] / TOP_N + daily[name] = net + d = {"gross": describe(gross), "net": describe(net), "stress": describe(stress), + "halves": [describe(net.loc[x:y]) for x, y in HALVES], + "post_pub": describe(net.loc[POST_PUB[0]:]), + "net_clipped_20pct": describe(clipped), + "share_of_gross_from_moves_over_10pct": float(big.sum() / gross.sum()), + "yearly": {int(k): float(v) for k, v in + ((1 + net).groupby(net.index.year).prod() - 1).items()}} + res["candidates"][name] = d + print(f"{name:15s} gross {d['gross']['mean_bp_per_day']:5.1f}bp/d sh {d['gross']['sharpe']:5.2f} | net " + f"cagr {d['net']['cagr']*100:5.1f}% vol {d['net']['vol']*100:4.1f}% sh {d['net']['sharpe']:5.2f} " + f"mdd {d['net']['mdd']*100:6.1f}% | halves {d['halves'][0]['sharpe']:5.2f} {d['halves'][1]['sharpe']:5.2f}" + f" | 2019+ {d['post_pub']['sharpe']:5.2f} | stress {d['stress']['sharpe']:5.2f} | clip " + f"{d['net_clipped_20pct']['sharpe']:5.2f} | >10% moves {d['share_of_gross_from_moves_over_10pct']:.2f}", + flush=True) + print(" yearly net %:", {k: round(v * 100, 1) for k, v in d["yearly"].items()}) + + uni = on.where(mem.shift(1).fillna(False).astype(bool)).mean(axis=1).loc[a:b] + res["universe_overnight_gross"] = {"full": describe(uni), "halves": [describe(uni.loc[x:y]) for x, y in HALVES], + "post_pub": describe(uni.loc[POST_PUB[0]:])} + u = res["universe_overnight_gross"] + print(f"EW members overnight gross: {u['full']['mean_bp_per_day']:.1f}bp/d sh {u['full']['sharpe']:.2f} " + f"halves {u['halves'][0]['sharpe']:.2f} {u['halves'][1]['sharpe']:.2f} 2019+ {u['post_pub']['sharpe']:.2f}") + + split = held_split(scores["sleeve_score"], on, intra) + res["sleeve_held_10d_split_gross"] = {} + for k, r in split.items(): + r = r.loc[a:b] + res["sleeve_held_10d_split_gross"][k] = {"full": describe(r), "halves": [describe(r.loc[x:y]) for x, y in HALVES], + "post_pub": describe(r.loc[POST_PUB[0]:])} + s = res["sleeve_held_10d_split_gross"][k] + print(f"sleeve held 10d, {k:14s}: {s['full']['mean_bp_per_day']:5.1f}bp/d cagr {s['full']['cagr']*100:5.1f}% " + f"sh {s['full']['sharpe']:5.2f} halves {s['halves'][0]['sharpe']:5.2f} {s['halves'][1]['sharpe']:5.2f} " + f"2019+ {s['post_pub']['sharpe']:5.2f}") + + fid = fidelity(p, sels) + res["fidelity"] = fid + print("fidelity:", {k: (round(v, 3) if isinstance(v, float) else v) for k, v in fid.items()}) + + n5 = (fid.get("symbols", 0) > 0 and fid["corr_all"] >= 0.9 + and all(abs(fid[f"{c}_mean_diff_bp"]) <= 3 for c in scores if fid.get(f"{c}_name_days", 0) > 0)) + checks = {} + for name, d in res["candidates"].items(): + checks[name] = {"N1": d["net"]["sharpe"] >= 1.5, "N2": all(h["sharpe"] >= 1.2 for h in d["halves"]), + "N3": d["post_pub"]["sharpe"] >= 1.0, "N4": d["stress"]["sharpe"] >= 1.0} + passing = [n for n, c in checks.items() if all(c.values())] + decision = "DATA" if not n5 else ("CONTINUE" if passing else "STOP") + res["verdict"] = {"checks": checks, "N5": n5, "passing": passing, "decision": decision} + print(f"checks {checks}\nN5 {n5} passing {passing} -> {decision}") + with open(res_path, "w") as fh: + json.dump(res, fh, indent=2) + pd.DataFrame(daily).to_pickle(os.path.join(OUT, "daily.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join(RESULTS, "trials_us.jsonl"), + [{"name": f"overnight_only:{n}", **res["candidates"][n]["net"]} for n in scores], + {"study": "overnight_only", "role": "preregistered quick check"}) + + +if __name__ == "__main__": + main()