gahow/quant-trading
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c34367161d2088b6df6eef21c6b4d8dad2f2d5dd
quant-trading/overnight_paper.py
Gahow Wang c34367161d research: ORB (STOP), overnight-only momentum (STOP), Alpaca paper forward test 
- orb_stocks_in_play: 5-min bounds -70%/yr (conservative) vs +71%/yr
  (optimistic); the optimistic bound breaks even at 3.6 bp slippage a side.
- overnight_only: the momentum sleeve's names earn their return overnight
  (Sharpe 1.57) and nothing intraday; held close->open only, net Sharpe
  1.29-1.42 at 1 bp a side, below the 1.5 gate.
- overnight_paper.py + systemd unit: MOC buy / MOO sell of the top 10 on the
  Alpaca paper account from 2026-10-12, with a fills-vs-prints report.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 20:06:34 -07:00

463 lines
20 KiB
Python
Raw Blame History

"""
Forward test of the overnight-only momentum book on Alpaca's PAPER account
(research: research/lab/overnight_only.py). Each session it buys the top 10 of
overnight_recovery's score with market-on-close orders and sells every
position with market-on-open orders at the next session, so the book is flat
during the day. The test measures what the backtest cannot: fills against the
official close/open prints, and the forward record.
uv run python overnight_paper.py signal # today's list, no orders
uv run python overnight_paper.py buy [--dry-run] # MOC orders; between close-25 and close-11 min
uv run python overnight_paper.py sell [--dry-run] # MOO sells; between 19:00 ET and open-2 min
uv run python overnight_paper.py report # fills vs official prints, nightly returns
uv run python overnight_paper.py daemon # runs sell / buy / report on Alpaca's calendar
Rules (as in the study's sleeve_score candidate):
score strategies.robust_momentum.signal_overnight_recovery (12-1 month
overnight momentum + distance from the 126-day low, percentile
blend) on Alpaca SIP daily bars (split/dividend adjusted) through
the previous session, current S&P 500 members
book top 10, equal weight, whole shares (Alpaca accepts only whole-share
MOC/MOO orders), 1x the account equity, no leverage
orders buy: market, time_in_force "cls", sent at close - 20 min (Alpaca
rejects MOC orders after 15:50 ET); sell: market, "opg", sent at
open - 20 min. A position left over from a failed sell is resized
to the new target at the close.
Keys come from ALPACA_API_ENDPOINT / ALPACA_API_KEY / ALPACA_API_SECRET in the
environment or .env; they are never printed. The script refuses any endpoint
that is not paper-api.alpaca.markets.
Boundary: Alpaca's paper engine simulates fills from live quotes; it does not
reproduce the exchange auctions or market impact. The report therefore scores
both the paper fills and the official open/close prints of the same names.
State and logs: data/overnight_paper/ (orders.jsonl, signals/<date>.json,
state.json, fills.csv, nights.csv).
"""
from __future__ import annotations
import argparse
import json
import math
import os
import sys
import time
from datetime import date, datetime, timedelta, timezone
from zoneinfo import ZoneInfo
import pandas as pd
import requests
from strategies.robust_momentum import signal_overnight_recovery
OUT = os.path.join("data", "overnight_paper")
DATA_URL = "https://data.alpaca.markets/v2"
PAPER_HOST = "paper-api.alpaca.markets"
NY = ZoneInfo("America/New_York")
TOP_N = 10
HISTORY_DAYS = 450 # calendar days of daily bars (score needs ~280 sessions)
BUY_BEFORE_CLOSE = timedelta(minutes=20)
BUY_CUTOFF = timedelta(minutes=11) # Alpaca: MOC orders close at 15:50 ET
SELL_BEFORE_OPEN = timedelta(minutes=20)
SELL_CUTOFF = timedelta(minutes=2) # Alpaca: MOO orders close at 09:28 ET
REPORT_AFTER_OPEN = timedelta(minutes=30)
# ---------------------------------------------------------------------------
# Alpaca client (paper only; keys never printed)
# ---------------------------------------------------------------------------
def _env() -> dict[str, str]:
env = dict(os.environ)
if os.path.exists(".env"):
for line in open(".env"):
if "=" in line and not line.lstrip().startswith("#"):
k, v = line.split("=", 1)
env.setdefault(k.strip(), v.strip().strip("'\""))
return env
class Alpaca:
def __init__(self) -> None:
env = _env()
base = env["ALPACA_API_ENDPOINT"].rstrip("/")
if PAPER_HOST not in base:
raise SystemExit("ALPACA_API_ENDPOINT is not the paper API; refusing to trade")
self.base = base if base.endswith("/v2") else base + "/v2"
self.s = requests.Session()
self.s.headers.update({"APCA-API-KEY-ID": env["ALPACA_API_KEY"],
"APCA-API-SECRET-KEY": env["ALPACA_API_SECRET"]})
def _req(self, method: str, url: str, **kw) -> requests.Response:
for attempt in range(6):
try:
r = self.s.request(method, url, timeout=30, **kw)
except requests.RequestException:
time.sleep(2 ** attempt)
continue
if r.status_code == 429 or r.status_code >= 500:
time.sleep(2 ** attempt)
continue
return r
raise RuntimeError(f"{method} {url.split('/v2')[-1]}: retries exhausted")
def get(self, path: str, data: bool = False, **params):
r = self._req("GET", (DATA_URL if data else self.base) + path, params=params)
if r.status_code != 200:
raise RuntimeError(f"GET {path}: HTTP {r.status_code} {r.text[:200]}")
return r.json()
def order(self, payload: dict) -> dict:
r = self._req("POST", self.base + "/orders", json=payload)
body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {"text": r.text}
return {"http": r.status_code, **body}
def cancel(self, order_id: str) -> int:
return self._req("DELETE", f"{self.base}/orders/{order_id}").status_code
def alpaca_symbol(yahoo: str) -> str:
return yahoo.replace("-", ".")
# ---------------------------------------------------------------------------
# calendar
# ---------------------------------------------------------------------------
def sessions(cl: Alpaca, start: date, end: date) -> pd.DataFrame:
rows = cl.get("/calendar", start=start.isoformat(), end=end.isoformat())
out = []
for c in rows:
d = date.fromisoformat(c["date"])
o = datetime.combine(d, datetime.strptime(c["open"], "%H:%M").time(), NY)
k = datetime.combine(d, datetime.strptime(c["close"], "%H:%M").time(), NY)
out.append({"date": d, "open": o, "close": k})
return pd.DataFrame(out).set_index("date")
def now_utc() -> datetime:
return datetime.now(timezone.utc)
# ---------------------------------------------------------------------------
# signal
# ---------------------------------------------------------------------------
def sp500() -> list[str]:
try:
from universe import get_sp500
return get_sp500()
except Exception as exc: # noqa: BLE001 - Wikipedia unreachable: last cached list
with open(os.path.join("data", "universe_sp500.json")) as fh:
cached = json.load(fh)
print(f"S&P 500 list from cache dated {cached.get('date')} ({type(exc).__name__})", flush=True)
return cached["tickers"]
def _data_end(end: date) -> str:
"""The free plan refuses SIP data from the last 15 minutes; a bare date counts as that whole day."""
stop = min(datetime.combine(end, datetime.strptime("23:59", "%H:%M").time(), NY), # daily bars are stamped 00:00 ET
now_utc() - timedelta(minutes=20))
return stop.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def daily_bars(cl: Alpaca, symbols: list[str], start: date, end: date) -> dict[str, pd.DataFrame]:
"""Adjusted SIP daily open/close, one column per Yahoo-spelled symbol."""
rows = []
amap = {alpaca_symbol(s): s for s in symbols}
names = list(amap)
for i in range(0, len(names), 100):
params = {"symbols": ",".join(names[i:i + 100]), "timeframe": "1Day", "start": start.isoformat(),
"end": _data_end(end), "adjustment": "all", "feed": "sip", "limit": 10000}
while True:
j = cl.get("/stocks/bars", data=True, **params)
for sym, bars in (j.get("bars") or {}).items():
for b in bars:
rows.append((amap[sym], b["t"][:10], b["o"], b["c"]))
tok = j.get("next_page_token")
if not tok:
break
params["page_token"] = tok
df = pd.DataFrame(rows, columns=["sym", "date", "open", "close"])
df["date"] = pd.to_datetime(df["date"])
return {f: df.pivot(index="date", columns="sym", values=f).sort_index() for f in ("open", "close")}
def compute_signal(cl: Alpaca, session: date) -> dict:
"""Top 10 for the close of ``session`` from bars through the previous session."""
cal = sessions(cl, session - timedelta(days=10), session)
prev = [d for d in cal.index if d < session]
if not prev:
raise RuntimeError(f"no session before {session}")
last = prev[-1]
syms = sp500()
bars = daily_bars(cl, syms, session - timedelta(days=HISTORY_DAYS), last)
if bars["close"].index[-1].date() != last:
raise RuntimeError(f"daily bars end {bars['close'].index[-1].date()}, expected {last}")
score = signal_overnight_recovery({"open": bars["open"], "close": bars["close"]})
row = score.iloc[-1].dropna().sort_values(ascending=False, kind="mergesort")
top = list(row.index[:TOP_N])
return {"session": session.isoformat(), "bars_through": last.isoformat(), "universe": len(syms),
"scored": int(row.size), "top": top, "scores": {s: round(float(row[s]), 4) for s in top},
"prev_close": {s: float(bars["close"][s].iloc[-1]) for s in top}}
# ---------------------------------------------------------------------------
# state / logs
# ---------------------------------------------------------------------------
def _state() -> dict:
path = os.path.join(OUT, "state.json")
return json.load(open(path)) if os.path.exists(path) else {}
def _save_state(st: dict) -> None:
os.makedirs(OUT, exist_ok=True)
tmp = os.path.join(OUT, ".state.json.tmp")
json.dump(st, open(tmp, "w"), indent=1)
os.replace(tmp, os.path.join(OUT, "state.json"))
def _log_order(kind: str, payload: dict, resp: dict) -> None:
os.makedirs(OUT, exist_ok=True)
keep = {k: resp.get(k) for k in ("http", "id", "client_order_id", "status", "symbol", "qty", "side",
"time_in_force", "submitted_at", "message", "text")}
with open(os.path.join(OUT, "orders.jsonl"), "a") as fh:
fh.write(json.dumps({"logged_at": now_utc().isoformat(), "kind": kind, "payload": payload,
"response": keep}) + "\n")
def _submit(cl: Alpaca, kind: str, payload: dict, dry: bool) -> None:
tag = f"{payload['side']:4s} {payload['qty']:>6s} {payload['symbol']:6s} {payload['time_in_force']}"
if dry:
print(f" [dry-run] {tag}", flush=True)
return
resp = cl.order(payload)
_log_order(kind, payload, resp)
ok = resp["http"] in (200, 201)
dup = resp["http"] == 422 and "client_order_id" in json.dumps(resp).lower()
print(f" {tag} -> {'ok ' + str(resp.get('status')) if ok else 'already sent' if dup else 'HTTP ' + str(resp['http']) + ' ' + str(resp.get('message', resp.get('text', '')))[:120]}",
flush=True)
# ---------------------------------------------------------------------------
# actions
# ---------------------------------------------------------------------------
def positions(cl: Alpaca) -> dict[str, int]:
return {p["symbol"]: int(float(p["qty"])) for p in cl.get("/positions")}
def open_orders(cl: Alpaca) -> list[dict]:
return cl.get("/orders", status="open", limit=500)
def latest_prices(cl: Alpaca, symbols: list[str]) -> dict[str, float]:
"""Last trade on the free real-time feed (IEX); sizing only."""
j = cl.get("/stocks/trades/latest", data=True, symbols=",".join(alpaca_symbol(s) for s in symbols), feed="iex")
return {s: float(j["trades"][alpaca_symbol(s)]["p"]) for s in symbols if alpaca_symbol(s) in j.get("trades", {})}
def buy(cl: Alpaca, dry: bool = False, force: bool = False) -> None:
today = datetime.now(NY).date()
cal = sessions(cl, today, today)
if today not in cal.index:
print(f"{today}: no session", flush=True)
return
close = cal.loc[today, "close"]
if not force and not (close - BUY_BEFORE_CLOSE - timedelta(minutes=5) <= now_utc() <= close - BUY_CUTOFF):
print(f"{today}: outside the MOC window (close {close:%H:%M} ET); use --force for a dry check", flush=True)
if not dry:
return
st = _state()
if st.get("bought") == today.isoformat() and not dry:
print(f"{today}: MOC orders already sent", flush=True)
return
sig = compute_signal(cl, today)
os.makedirs(os.path.join(OUT, "signals"), exist_ok=True)
json.dump(sig, open(os.path.join(OUT, "signals", f"{today}.json"), "w"), indent=1)
acct = cl.get("/account")
equity = float(acct["equity"])
px = latest_prices(cl, sig["top"])
held = positions(cl)
pending = {o["symbol"] for o in open_orders(cl) if o["time_in_force"] == "cls"}
target = {}
for s in sig["top"]:
p = px.get(s) or sig["prev_close"][s]
target[alpaca_symbol(s)] = math.floor(equity / TOP_N / p)
print(f"{today}: equity {equity:,.0f}, bars through {sig['bars_through']}, top {sig['top']}", flush=True)
for sym in sorted(set(target) | set(held)):
if sym in pending:
print(f" {sym}: MOC order already open", flush=True)
continue
delta = target.get(sym, 0) - held.get(sym, 0)
if delta == 0:
continue
payload = {"symbol": sym, "qty": str(abs(delta)), "side": "buy" if delta > 0 else "sell",
"type": "market", "time_in_force": "cls", "client_order_id": f"onb-{today}-{sym}"}
_submit(cl, "moc", payload, dry)
if not dry:
st["bought"] = today.isoformat()
_save_state(st)
def sell(cl: Alpaca, dry: bool = False) -> None:
now = now_utc()
cal = sessions(cl, (now - timedelta(days=1)).date(), (now + timedelta(days=10)).date())
nxt = cal[cal["open"] > now]
if nxt.empty:
print("no upcoming session", flush=True)
return
session, opn = nxt.index[0], nxt["open"].iloc[0]
if not dry and now > opn - SELL_CUTOFF:
print(f"{session}: too late for MOO orders", flush=True)
return
st = _state()
if st.get("sold") == session.isoformat() and not dry:
print(f"{session}: MOO orders already sent", flush=True)
return
held = {s: q for s, q in positions(cl).items() if q > 0}
pending = {o["symbol"] for o in open_orders(cl) if o["time_in_force"] == "opg" and o["side"] == "sell"}
print(f"{session}: MOO sells for {len(held)} positions", flush=True)
for sym, q in sorted(held.items()):
if sym in pending:
print(f" {sym}: MOO order already open", flush=True)
continue
payload = {"symbol": sym, "qty": str(q), "side": "sell", "type": "market", "time_in_force": "opg",
"client_order_id": f"ons-{session}-{sym}"}
_submit(cl, "moo", payload, dry)
if not dry:
st["sold"] = session.isoformat()
_save_state(st)
def report(cl: Alpaca) -> None:
"""Fills against the official prints (Alpaca SIP daily open/close, unadjusted) and nightly returns."""
orders, after = [], (now_utc() - timedelta(days=400)).isoformat()
while True:
page = cl.get("/orders", status="closed", limit=500, direction="asc", after=after)
orders += page
if len(page) < 500:
break
after = page[-1]["submitted_at"]
rows = []
for o in orders:
cid = o.get("client_order_id", "")
if not cid.startswith(("onb-", "ons-")) or o.get("filled_avg_price") is None:
continue
rows.append({"session": cid[4:14], "symbol": o["symbol"], "side": o["side"], "tif": o["time_in_force"],
"qty": float(o["filled_qty"]), "fill": float(o["filled_avg_price"]),
"filled_at": o["filled_at"]})
if not rows:
print("no fills yet", flush=True)
return
F = pd.DataFrame(rows)
d0 = date.fromisoformat(F["session"].min())
d1 = date.fromisoformat(F["session"].max())
rows = []
syms = sorted(F["symbol"].unique())
for i in range(0, len(syms), 100):
params = {"symbols": ",".join(syms[i:i + 100]), "timeframe": "1Day", "start": d0.isoformat(),
"end": _data_end(d1), "adjustment": "raw", "feed": "sip", "limit": 10000}
while True:
j = cl.get("/stocks/bars", data=True, **params)
for sym, bars in (j.get("bars") or {}).items():
rows += [(sym, b["t"][:10], b["o"], b["c"]) for b in bars]
if not j.get("next_page_token"):
break
params["page_token"] = j["next_page_token"]
P = pd.DataFrame(rows, columns=["symbol", "session", "open", "close"])
F = F.merge(P, on=["symbol", "session"], how="left")
F["ref"] = F["close"].where(F["tif"] == "cls", F["open"])
# cost in bp: positive = worse than the official print
sign = F["side"].map({"buy": 1, "sell": -1})
F["slip_bp"] = sign * (F["fill"] / F["ref"] - 1) * 1e4
F.to_csv(os.path.join(OUT, "fills.csv"), index=False)
b = F[F["tif"] == "cls"]
s = F[F["tif"] == "opg"]
print(f"fills {len(F)}: MOC slip {b['slip_bp'].mean():.2f} bp (median {b['slip_bp'].median():.2f}), "
f"MOO slip {s['slip_bp'].mean():.2f} bp (median {s['slip_bp'].median():.2f})", flush=True)
# nights: buy session d (close) -> sell next session (open)
nights = []
for d, g in b.groupby("session"):
later = s[s["session"] > d]
if later.empty:
continue
nd = later["session"].min()
sg = later[later["session"] == nd].set_index("symbol")
bg = g.set_index("symbol")
both = bg.index.intersection(sg.index)
if both.empty:
continue
cost = (bg.loc[both, "qty"] * bg.loc[both, "fill"]).sum()
pnl = (sg.loc[both, "qty"] * sg.loc[both, "fill"] - bg.loc[both, "qty"] * bg.loc[both, "fill"]).sum()
model = (sg.loc[both, "open"] / bg.loc[both, "close"] - 1).mean()
nights.append({"buy_session": d, "sell_session": nd, "names": len(both),
"paper_ret": pnl / cost, "print_ret": model})
if nights:
N = pd.DataFrame(nights)
N.to_csv(os.path.join(OUT, "nights.csv"), index=False)
print(N.tail(10).to_string(index=False, float_format=lambda x: f"{x:.4f}"), flush=True)
print(f"nights {len(N)}: paper mean {N['paper_ret'].mean()*1e4:.1f} bp/night, "
f"prints mean {N['print_ret'].mean()*1e4:.1f} bp/night", flush=True)
# ---------------------------------------------------------------------------
# daemon
# ---------------------------------------------------------------------------
def daemon() -> None:
cl = Alpaca()
print(f"{now_utc():%Y-%m-%d %H:%M} UTC daemon started (paper account)", flush=True)
done: set[tuple[str, str]] = set()
while True:
try:
now = now_utc()
cal = sessions(cl, (now - timedelta(days=1)).date(), (now + timedelta(days=10)).date())
for d, row in cal.iterrows():
key = d.isoformat()
sell_at, buy_at = row["open"] - SELL_BEFORE_OPEN, row["close"] - BUY_BEFORE_CLOSE
if ("sell", key) not in done and sell_at <= now < row["open"] - SELL_CUTOFF:
sell(cl)
done.add(("sell", key))
if ("buy", key) not in done and buy_at <= now < row["close"] - BUY_CUTOFF:
buy(cl)
done.add(("buy", key))
if ("report", key) not in done and row["open"] + REPORT_AFTER_OPEN <= now < row["close"]:
report(cl)
done.add(("report", key))
except Exception as exc: # noqa: BLE001 - keep the loop alive; the next tick retries
print(f"{now_utc():%Y-%m-%d %H:%M} UTC error: {type(exc).__name__}: {str(exc)[:300]}", flush=True)
time.sleep(60)
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("what", choices=["signal", "buy", "sell", "report", "daemon"])
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--force", action="store_true", help="buy: ignore the time window (still honours --dry-run)")
a = ap.parse_args()
if a.what == "daemon":
daemon()
return
cl = Alpaca()
if a.what == "signal":
today = datetime.now(NY).date()
cal = sessions(cl, today, today + timedelta(days=10))
nxt = cal[cal["close"] > now_utc()].index[0]
sig = compute_signal(cl, nxt)
print(json.dumps(sig, indent=1))
elif a.what == "buy":
if a.force and not a.dry_run:
sys.exit("--force is only for --dry-run checks")
buy(cl, dry=a.dry_run, force=a.force)
elif a.what == "sell":
sell(cl, dry=a.dry_run)
else:
report(cl)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink