"""
Forward test of the overnight-only momentum book on Alpaca's PAPER account
(research: research/lab/overnight_only.py). Each session it buys the top 10 of
overnight_recovery's score with market-on-close orders and sells every
position with market-on-open orders at the next session, so the book is flat
during the day. The test measures what the backtest cannot: fills against the
official close/open prints, and the forward record.

    uv run python overnight_paper.py signal            # today's list, no orders
    uv run python overnight_paper.py buy  [--dry-run]  # MOC orders; between close-25 and close-11 min
    uv run python overnight_paper.py sell [--dry-run]  # MOO sells; between 19:00 ET and open-2 min
    uv run python overnight_paper.py report            # fills vs official prints, nightly returns
    uv run python overnight_paper.py daemon            # runs sell / buy / report on Alpaca's calendar

Rules (as in the study's sleeve_score candidate):
  score      strategies.robust_momentum.signal_overnight_recovery (12-1 month
             overnight momentum + distance from the 126-day low, percentile
             blend) on Alpaca SIP daily bars (split/dividend adjusted) through
             the previous session, current S&P 500 members
  book       top 10, equal weight, whole shares (Alpaca accepts only whole-share
             MOC/MOO orders), 1x the account equity, no leverage
  orders     buy: market, time_in_force "cls", sent at close - 20 min (Alpaca
             rejects MOC orders after 15:50 ET); sell: market, "opg", sent at
             open - 20 min. A position left over from a failed sell is resized
             to the new target at the close.
Keys come from ALPACA_API_ENDPOINT / ALPACA_API_KEY / ALPACA_API_SECRET in the
environment or .env; they are never printed. The script refuses any endpoint
that is not paper-api.alpaca.markets.

Boundary: Alpaca's paper engine simulates fills from live quotes; it does not
reproduce the exchange auctions or market impact. The report therefore scores
both the paper fills and the official open/close prints of the same names.

State and logs: data/overnight_paper/ (orders.jsonl, signals/<date>.json,
state.json, fills.csv, nights.csv).
"""

from __future__ import annotations

import argparse
import json
import math
import os
import sys
import time
from datetime import date, datetime, timedelta, timezone
from zoneinfo import ZoneInfo

import pandas as pd
import requests

from strategies.robust_momentum import signal_overnight_recovery

OUT = os.path.join("data", "overnight_paper")
DATA_URL = "https://data.alpaca.markets/v2"
PAPER_HOST = "paper-api.alpaca.markets"
NY = ZoneInfo("America/New_York")
TOP_N = 10
HISTORY_DAYS = 450                      # calendar days of daily bars (score needs ~280 sessions)
BUY_BEFORE_CLOSE = timedelta(minutes=20)
BUY_CUTOFF = timedelta(minutes=11)      # Alpaca: MOC orders close at 15:50 ET
SELL_BEFORE_OPEN = timedelta(minutes=20)
SELL_CUTOFF = timedelta(minutes=2)      # Alpaca: MOO orders close at 09:28 ET
REPORT_AFTER_OPEN = timedelta(minutes=30)
ACCEPTED_ORDER_STATUSES = {"accepted", "pending_new", "new", "accepted_for_bidding", "partially_filled", "filled"}


# ---------------------------------------------------------------------------
# Alpaca client (paper only; keys never printed)
# ---------------------------------------------------------------------------

def _env() -> dict[str, str]:
    env = dict(os.environ)
    if os.path.exists(".env"):
        for line in open(".env"):
            if "=" in line and not line.lstrip().startswith("#"):
                k, v = line.split("=", 1)
                env.setdefault(k.strip(), v.strip().strip("'\""))
    return env


class Alpaca:
    def __init__(self) -> None:
        env = _env()
        base = env["ALPACA_API_ENDPOINT"].rstrip("/")
        if PAPER_HOST not in base:
            raise SystemExit("ALPACA_API_ENDPOINT is not the paper API; refusing to trade")
        self.base = base if base.endswith("/v2") else base + "/v2"
        self.s = requests.Session()
        self.s.headers.update({"APCA-API-KEY-ID": env["ALPACA_API_KEY"],
                               "APCA-API-SECRET-KEY": env["ALPACA_API_SECRET"]})

    def _req(self, method: str, url: str, **kw) -> requests.Response:
        for attempt in range(6):
            try:
                r = self.s.request(method, url, timeout=30, **kw)
            except requests.RequestException:
                time.sleep(2 ** attempt)
                continue
            if r.status_code == 429 or r.status_code >= 500:
                time.sleep(2 ** attempt)
                continue
            return r
        raise RuntimeError(f"{method} {url.split('/v2')[-1]}: retries exhausted")

    def get(self, path: str, data: bool = False, **params):
        r = self._req("GET", (DATA_URL if data else self.base) + path, params=params)
        if r.status_code != 200:
            raise RuntimeError(f"GET {path}: HTTP {r.status_code} {r.text[:200]}")
        return r.json()

    def order(self, payload: dict) -> dict:
        r = self._req("POST", self.base + "/orders", json=payload)
        body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {"text": r.text}
        return {"http": r.status_code, **body}

    def cancel(self, order_id: str) -> int:
        return self._req("DELETE", f"{self.base}/orders/{order_id}").status_code


def alpaca_symbol(yahoo: str) -> str:
    return yahoo.replace("-", ".")


# ---------------------------------------------------------------------------
# calendar
# ---------------------------------------------------------------------------

def sessions(cl: Alpaca, start: date, end: date) -> pd.DataFrame:
    rows = cl.get("/calendar", start=start.isoformat(), end=end.isoformat())
    out = []
    for c in rows:
        d = date.fromisoformat(c["date"])
        o = datetime.combine(d, datetime.strptime(c["open"], "%H:%M").time(), NY)
        k = datetime.combine(d, datetime.strptime(c["close"], "%H:%M").time(), NY)
        out.append({"date": d, "open": o, "close": k})
    return pd.DataFrame(out).set_index("date")


def now_utc() -> datetime:
    return datetime.now(timezone.utc)


# ---------------------------------------------------------------------------
# signal
# ---------------------------------------------------------------------------

def sp500() -> list[str]:
    try:
        from universe import get_sp500
        return get_sp500()
    except Exception as exc:  # noqa: BLE001 - Wikipedia unreachable: last cached list
        with open(os.path.join("data", "universe_sp500.json")) as fh:
            cached = json.load(fh)
        print(f"S&P 500 list from cache dated {cached.get('date')} ({type(exc).__name__})", flush=True)
        return cached["tickers"]


def _data_end(end: date) -> str:
    """The free plan refuses SIP data from the last 15 minutes; a bare date counts as that whole day."""
    stop = min(datetime.combine(end, datetime.strptime("23:59", "%H:%M").time(), NY),   # daily bars are stamped 00:00 ET
               now_utc() - timedelta(minutes=20))
    return stop.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def daily_bars(cl: Alpaca, symbols: list[str], start: date, end: date) -> dict[str, pd.DataFrame]:
    """Adjusted SIP daily open/close, one column per Yahoo-spelled symbol."""
    rows = []
    amap = {alpaca_symbol(s): s for s in symbols}
    names = list(amap)
    for i in range(0, len(names), 100):
        params = {"symbols": ",".join(names[i:i + 100]), "timeframe": "1Day", "start": start.isoformat(),
                  "end": _data_end(end), "adjustment": "all", "feed": "sip", "limit": 10000}
        while True:
            j = cl.get("/stocks/bars", data=True, **params)
            for sym, bars in (j.get("bars") or {}).items():
                for b in bars:
                    rows.append((amap[sym], b["t"][:10], b["o"], b["c"]))
            tok = j.get("next_page_token")
            if not tok:
                break
            params["page_token"] = tok
    df = pd.DataFrame(rows, columns=["sym", "date", "open", "close"])
    df["date"] = pd.to_datetime(df["date"])
    return {f: df.pivot(index="date", columns="sym", values=f).sort_index() for f in ("open", "close")}


def compute_signal(cl: Alpaca, session: date) -> dict:
    """Top 10 for the close of ``session`` from bars through the previous session."""
    cal = sessions(cl, session - timedelta(days=10), session)
    prev = [d for d in cal.index if d < session]
    if not prev:
        raise RuntimeError(f"no session before {session}")
    last = prev[-1]
    syms = sp500()
    bars = daily_bars(cl, syms, session - timedelta(days=HISTORY_DAYS), last)
    if bars["close"].index[-1].date() != last:
        raise RuntimeError(f"daily bars end {bars['close'].index[-1].date()}, expected {last}")
    score = signal_overnight_recovery({"open": bars["open"], "close": bars["close"]})
    row = score.iloc[-1].dropna().sort_values(ascending=False, kind="mergesort")
    top = list(row.index[:TOP_N])
    return {"session": session.isoformat(), "bars_through": last.isoformat(), "universe": len(syms),
            "scored": int(row.size), "top": top, "scores": {s: round(float(row[s]), 4) for s in top},
            "prev_close": {s: float(bars["close"][s].iloc[-1]) for s in top}}


# ---------------------------------------------------------------------------
# state / logs
# ---------------------------------------------------------------------------

def _state() -> dict:
    path = os.path.join(OUT, "state.json")
    if not os.path.exists(path):
        return {}
    with open(path) as fh:
        return json.load(fh)


def _save_state(st: dict) -> None:
    os.makedirs(OUT, exist_ok=True)
    tmp = os.path.join(OUT, ".state.json.tmp")
    with open(tmp, "w") as fh:
        json.dump(st, fh, indent=1)
    os.replace(tmp, os.path.join(OUT, "state.json"))


def _log_order(kind: str, payload: dict, resp: dict) -> None:
    os.makedirs(OUT, exist_ok=True)
    keep = {k: resp.get(k) for k in ("http", "id", "client_order_id", "status", "symbol", "qty", "side",
                                     "time_in_force", "submitted_at", "message", "text")}
    with open(os.path.join(OUT, "orders.jsonl"), "a") as fh:
        fh.write(json.dumps({"logged_at": now_utc().isoformat(), "kind": kind, "payload": payload,
                             "response": keep}) + "\n")


def _submit(cl: Alpaca, kind: str, payload: dict, dry: bool) -> None:
    tag = f"{payload['side']:4s} {payload['qty']:>6s} {payload['symbol']:6s} {payload['time_in_force']}"
    if dry:
        print(f"  [dry-run] {tag}", flush=True)
        return
    resp = cl.order(payload)
    _log_order(kind, payload, resp)
    if resp["http"] == 422 and "client_order_id" in str(resp.get("message", "")).lower():
        existing = cl.get("/orders:by_client_order_id", client_order_id=payload["client_order_id"])
        matches = all(existing.get(k) == payload[k] for k in ("client_order_id", "symbol", "side", "type", "time_in_force"))
        matches &= float(existing.get("qty", 0)) == float(payload["qty"])
        if matches and existing.get("id") and existing.get("status") in ACCEPTED_ORDER_STATUSES:
            print(f"  {tag} -> already accepted ({existing['status']})", flush=True)
            return
        raise RuntimeError(f"{tag}: client_order_id belongs to a mismatched or {existing.get('status')} order")
    if resp["http"] not in (200, 201) or not resp.get("id") or resp.get("status") not in ACCEPTED_ORDER_STATUSES:
        reason = str(resp.get("message", resp.get("text", resp.get("status", "missing order acknowledgement"))))[:200]
        raise RuntimeError(f"{tag}: HTTP {resp['http']} {reason}")
    print(f"  {tag} -> ok {resp['status']}", flush=True)


# ---------------------------------------------------------------------------
# actions
# ---------------------------------------------------------------------------

def positions(cl: Alpaca) -> dict[str, int]:
    return {p["symbol"]: int(float(p["qty"])) for p in cl.get("/positions")}


def open_orders(cl: Alpaca) -> list[dict]:
    return cl.get("/orders", status="open", limit=500)


def latest_prices(cl: Alpaca, symbols: list[str]) -> dict[str, float]:
    """Last trade on the free real-time feed (IEX); sizing only."""
    j = cl.get("/stocks/trades/latest", data=True, symbols=",".join(alpaca_symbol(s) for s in symbols), feed="iex")
    return {s: float(j["trades"][alpaca_symbol(s)]["p"]) for s in symbols if alpaca_symbol(s) in j.get("trades", {})}


def buy(cl: Alpaca, dry: bool = False, force: bool = False) -> None:
    today = datetime.now(NY).date()
    cal = sessions(cl, today, today)
    if today not in cal.index:
        print(f"{today}: no session", flush=True)
        return
    close = cal.loc[today, "close"]
    if not force and not (close - BUY_BEFORE_CLOSE - timedelta(minutes=5) <= now_utc() <= close - BUY_CUTOFF):
        print(f"{today}: outside the MOC window (close {close:%H:%M} ET); use --force for a dry check", flush=True)
        if not dry:
            return
    st = _state()
    if st.get("bought") == today.isoformat() and not dry:
        print(f"{today}: MOC orders already sent", flush=True)
        return
    sig = compute_signal(cl, today)
    os.makedirs(os.path.join(OUT, "signals"), exist_ok=True)
    with open(os.path.join(OUT, "signals", f"{today}.json"), "w") as fh:
        json.dump(sig, fh, indent=1)
    acct = cl.get("/account")
    equity = float(acct["equity"])
    px = latest_prices(cl, sig["top"])
    held = positions(cl)
    pending = {o["symbol"] for o in open_orders(cl) if o["time_in_force"] == "cls"}
    target = {}
    for s in sig["top"]:
        p = px.get(s) or sig["prev_close"][s]
        target[alpaca_symbol(s)] = math.floor(equity / TOP_N / p)
    print(f"{today}: equity {equity:,.0f}, bars through {sig['bars_through']}, top {sig['top']}", flush=True)
    for sym in sorted(set(target) | set(held)):
        if sym in pending:
            print(f"  {sym}: MOC order already open", flush=True)
            continue
        delta = target.get(sym, 0) - held.get(sym, 0)
        if delta == 0:
            continue
        payload = {"symbol": sym, "qty": str(abs(delta)), "side": "buy" if delta > 0 else "sell",
                   "type": "market", "time_in_force": "cls", "client_order_id": f"onb-{today}-{sym}"}
        _submit(cl, "moc", payload, dry)
    if not dry:
        st["bought"] = today.isoformat()
        _save_state(st)


def sell(cl: Alpaca, dry: bool = False) -> None:
    now = now_utc()
    cal = sessions(cl, (now - timedelta(days=1)).date(), (now + timedelta(days=10)).date())
    nxt = cal[cal["open"] > now]
    if nxt.empty:
        print("no upcoming session", flush=True)
        return
    session, opn = nxt.index[0], nxt["open"].iloc[0]
    if not dry and now > opn - SELL_CUTOFF:
        print(f"{session}: too late for MOO orders", flush=True)
        return
    st = _state()
    if st.get("sold") == session.isoformat() and not dry:
        print(f"{session}: MOO orders already sent", flush=True)
        return
    held = {s: q for s, q in positions(cl).items() if q > 0}
    pending = {o["symbol"] for o in open_orders(cl) if o["time_in_force"] == "opg" and o["side"] == "sell"}
    print(f"{session}: MOO sells for {len(held)} positions", flush=True)
    for sym, q in sorted(held.items()):
        if sym in pending:
            print(f"  {sym}: MOO order already open", flush=True)
            continue
        payload = {"symbol": sym, "qty": str(q), "side": "sell", "type": "market", "time_in_force": "opg",
                   "client_order_id": f"ons-{session}-{sym}"}
        _submit(cl, "moo", payload, dry)
    if not dry:
        st["sold"] = session.isoformat()
        _save_state(st)


def report(cl: Alpaca) -> None:
    """Fills against the official prints (Alpaca SIP daily open/close, unadjusted) and nightly returns."""
    orders, after = [], (now_utc() - timedelta(days=400)).isoformat()
    while True:
        page = cl.get("/orders", status="closed", limit=500, direction="asc", after=after)
        orders += page
        if len(page) < 500:
            break
        after = page[-1]["submitted_at"]
    rows = []
    for o in orders:
        cid = o.get("client_order_id", "")
        if not cid.startswith(("onb-", "ons-")) or o.get("filled_avg_price") is None:
            continue
        rows.append({"session": cid[4:14], "symbol": o["symbol"], "side": o["side"], "tif": o["time_in_force"],
                     "qty": float(o["filled_qty"]), "fill": float(o["filled_avg_price"]),
                     "filled_at": o["filled_at"]})
    if not rows:
        print("no fills yet", flush=True)
        return
    F = pd.DataFrame(rows)
    d0 = date.fromisoformat(F["session"].min())
    d1 = date.fromisoformat(F["session"].max())
    rows = []
    syms = sorted(F["symbol"].unique())
    for i in range(0, len(syms), 100):
        params = {"symbols": ",".join(syms[i:i + 100]), "timeframe": "1Day", "start": d0.isoformat(),
                  "end": _data_end(d1), "adjustment": "raw", "feed": "sip", "limit": 10000}
        while True:
            j = cl.get("/stocks/bars", data=True, **params)
            for sym, bars in (j.get("bars") or {}).items():
                rows += [(sym, b["t"][:10], b["o"], b["c"]) for b in bars]
            if not j.get("next_page_token"):
                break
            params["page_token"] = j["next_page_token"]
    P = pd.DataFrame(rows, columns=["symbol", "session", "open", "close"])
    F = F.merge(P, on=["symbol", "session"], how="left")
    F["ref"] = F["close"].where(F["tif"] == "cls", F["open"])
    # cost in bp: positive = worse than the official print
    sign = F["side"].map({"buy": 1, "sell": -1})
    F["slip_bp"] = sign * (F["fill"] / F["ref"] - 1) * 1e4
    F.to_csv(os.path.join(OUT, "fills.csv"), index=False)
    b = F[F["tif"] == "cls"]
    s = F[F["tif"] == "opg"]
    print(f"fills {len(F)}: MOC slip {b['slip_bp'].mean():.2f} bp (median {b['slip_bp'].median():.2f}), "
          f"MOO slip {s['slip_bp'].mean():.2f} bp (median {s['slip_bp'].median():.2f})", flush=True)
    # nights: buy session d (close) -> sell next session (open)
    nights = []
    for d, g in b.groupby("session"):
        later = s[s["session"] > d]
        if later.empty:
            continue
        nd = later["session"].min()
        sg = later[later["session"] == nd].set_index("symbol")
        bg = g.set_index("symbol")
        both = bg.index.intersection(sg.index)
        if both.empty:
            continue
        cost = (bg.loc[both, "qty"] * bg.loc[both, "fill"]).sum()
        pnl = (sg.loc[both, "qty"] * sg.loc[both, "fill"] - bg.loc[both, "qty"] * bg.loc[both, "fill"]).sum()
        model = (sg.loc[both, "open"] / bg.loc[both, "close"] - 1).mean()
        nights.append({"buy_session": d, "sell_session": nd, "names": len(both),
                       "paper_ret": pnl / cost, "print_ret": model})
    if nights:
        N = pd.DataFrame(nights)
        N.to_csv(os.path.join(OUT, "nights.csv"), index=False)
        print(N.tail(10).to_string(index=False, float_format=lambda x: f"{x:.4f}"), flush=True)
        print(f"nights {len(N)}: paper mean {N['paper_ret'].mean()*1e4:.1f} bp/night, "
              f"prints mean {N['print_ret'].mean()*1e4:.1f} bp/night", flush=True)


# ---------------------------------------------------------------------------
# daemon
# ---------------------------------------------------------------------------

def daemon() -> None:
    cl = Alpaca()
    print(f"{now_utc():%Y-%m-%d %H:%M} UTC daemon started (paper account)", flush=True)
    done: set[tuple[str, str]] = set()
    while True:
        try:
            now = now_utc()
            cal = sessions(cl, (now - timedelta(days=1)).date(), (now + timedelta(days=10)).date())
            for d, row in cal.iterrows():
                key = d.isoformat()
                sell_at, buy_at = row["open"] - SELL_BEFORE_OPEN, row["close"] - BUY_BEFORE_CLOSE
                if ("sell", key) not in done and sell_at <= now < row["open"] - SELL_CUTOFF:
                    sell(cl)
                    done.add(("sell", key))
                if ("buy", key) not in done and buy_at <= now < row["close"] - BUY_CUTOFF:
                    buy(cl)
                    done.add(("buy", key))
                if ("report", key) not in done and row["open"] + REPORT_AFTER_OPEN <= now < row["close"]:
                    report(cl)
                    done.add(("report", key))
        except Exception as exc:  # noqa: BLE001 - keep the loop alive; the next tick retries
            print(f"{now_utc():%Y-%m-%d %H:%M} UTC error: {type(exc).__name__}: {str(exc)[:300]}", flush=True)
        time.sleep(60)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("what", choices=["signal", "buy", "sell", "report", "daemon"])
    ap.add_argument("--dry-run", action="store_true")
    ap.add_argument("--force", action="store_true", help="buy: ignore the time window (still honours --dry-run)")
    a = ap.parse_args()
    if a.what == "daemon":
        daemon()
        return
    cl = Alpaca()
    if a.what == "signal":
        today = datetime.now(NY).date()
        cal = sessions(cl, today, today + timedelta(days=10))
        nxt = cal[cal["close"] > now_utc()].index[0]
        sig = compute_signal(cl, nxt)
        print(json.dumps(sig, indent=1))
    elif a.what == "buy":
        if a.force and not a.dry_run:
            sys.exit("--force is only for --dry-run checks")
        buy(cl, dry=a.dry_run, force=a.force)
    elif a.what == "sell":
        sell(cl, dry=a.dry_run)
    else:
        report(cl)


if __name__ == "__main__":
    main()