"""
Long daily history for the leveraged-ETF and cross-asset research
(research/lab/letf_history.py): stock indices from 1927/1985, the ETFs the
trend strategies trade, gold futures, bitcoin, and Treasury yields.

    PYTHONPATH=. uv run python -m research.lab.fetch_long_history [--no-proxy]

Yahoo is fetched ticker by ticker (batched requests fail intermittently)
and each ticker is cached in data/research/ext/yahoo/ as it arrives, so a
re-run after rate limiting fetches only what is missing. The fetch stops
after three tickers fail in a row: Yahoo is rate limiting, and the live
monitor needs it too. FRED yields come from the public fredgraph CSV
endpoint. Writes data/research/long_history.pkl: {"close", "open"}
DataFrames (adjusted for splits and dividends) and "fred" (yields in
percent, and the Nasdaq-100 level).
"""

from __future__ import annotations

import argparse
import io
import os
import pickle
import time
import urllib.request
import warnings

import pandas as pd
import yfinance as yf

from research.lab.fetch_us import _proxy_env

warnings.filterwarnings("ignore")
OUT = os.path.join("data", "research", "long_history.pkl")
YAHOO = [
    "^GSPC", "^NDX", "^VIX",                                   # indices (price only)
    "SPY", "QQQ", "IWM", "EFA", "EEM", "VNQ", "TLT", "IEF", "SHY", "GLD", "DBC", "GSG", "UUP",
    "TQQQ", "UPRO", "QLD", "SSO", "SOXL", "TMF", "UGL",        # leveraged ETFs (for validation)
    "VFINX", "VUSTX",                                           # mutual funds with dividends, 1980s-
    "GC=F", "BTC-USD", "ETH-USD", "IBIT",
]
FRED = ["DTB3", "DGS2", "DGS10", "DGS30", "NASDAQ100"]


CACHE = os.path.join("data", "research", "ext", "yahoo")


def _cached(t: str) -> str:
    return os.path.join(CACHE, t.replace("^", "_").replace("=", "_") + ".csv")


def fetch_yahoo(tickers: list[str], start: str = "1927-01-01") -> tuple[pd.DataFrame, pd.DataFrame]:
    os.makedirs(CACHE, exist_ok=True)
    fails = 0
    for t in tickers:
        if os.path.exists(_cached(t)) or fails >= 3:
            continue
        for attempt in range(2):
            try:
                h = yf.Ticker(t).history(start=start, auto_adjust=True)
                if len(h):
                    h.index = h.index.tz_localize(None).normalize()
                    h[["Open", "Close"]].to_csv(_cached(t))
                    break
            except Exception as exc:  # noqa: BLE001 — retry transient Yahoo errors
                print(f"  {t}: {type(exc).__name__}", flush=True)
            time.sleep(20)
        fails = 0 if os.path.exists(_cached(t)) else fails + 1
        print(f"{t:8s} {'ok' if os.path.exists(_cached(t)) else 'MISSING'}", flush=True)
        time.sleep(2)
    close, opens = {}, {}
    for t in tickers:
        if os.path.exists(_cached(t)):
            h = pd.read_csv(_cached(t), index_col=0, parse_dates=True)
            close[t], opens[t] = h["Close"], h["Open"]
    missing = [t for t in tickers if t not in close]
    if missing:
        print("missing (re-run later):", missing)
    return pd.DataFrame(close).sort_index(), pd.DataFrame(opens).sort_index()


def fetch_fred(series: list[str]) -> pd.DataFrame:
    out = {}
    for s in series:
        url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={s}"
        with urllib.request.urlopen(url, timeout=60) as fh:
            df = pd.read_csv(io.BytesIO(fh.read()), index_col=0, parse_dates=True)
        out[s] = pd.to_numeric(df.iloc[:, 0], errors="coerce")
    return pd.DataFrame(out)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--no-proxy", action="store_true",
                    help="connect directly (when the proxy's IP is rate limited by Yahoo)")
    if not ap.parse_args().no_proxy:
        _proxy_env()
    close, opens = fetch_yahoo(YAHOO)
    fred = fetch_fred(FRED)
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    with open(OUT, "wb") as fh:
        pickle.dump({"close": close, "open": opens, "fred": fred}, fh)
    print(f"Saved {OUT}: {close.shape[1]} series, FRED {list(fred.columns)}")


if __name__ == "__main__":
    main()