quant-trading/research/lab/fetch_tiingo.py

"""Fetch EOD prices from Tiingo (free tier: 500 unique symbols/month, 50/hour).

Used for: (1) completing delisted/missing names in the research panels,
(2) independent cross-validation of the Yahoo-sourced closes. The key comes
from TIINGO_API_KEY (environment or .env) and is never logged or written.

Writes data/research/tiingo/<ticker>.csv (resume-safe; a cached symbol never
counts against the monthly quota again) and <ticker>.miss for symbols Tiingo
does not serve (also skipped on later runs, so dead names do not re-burn the
500-symbol monthly quota). Columns: as-traded open/high/low/close/volume
(Tiingo's close is NOT split-adjusted), split/dividend-adjusted
adjOpen..adjVolume, plus splitFactor and divCash.

    PYTHONPATH=. uv run python -m research.lab.fetch_tiingo TICKER [TICKER ...]
    PYTHONPATH=. uv run python -m research.lab.fetch_tiingo --file symbols.txt
    PYTHONPATH=. uv run python -m research.lab.fetch_tiingo --file l.txt --min-interval 75
"""
from __future__ import annotations

import argparse
import os
import re
import sys
import time

import pandas as pd
import requests

DIR = os.path.join("data", "research", "tiingo")
BASE = "https://api.tiingo.com/tiingo/daily/{t}/prices"
KEEP = ("date", "open", "high", "low", "close", "volume",
        "adjOpen", "adjHigh", "adjLow", "adjClose", "adjVolume",
        "splitFactor", "divCash")


def api_key() -> str:
    key = os.environ.get("TIINGO_API_KEY")
    if not key and os.path.exists(".env"):
        for line in open(".env"):
            m = re.match(r"\s*TIINGO_API_KEY\s*=\s*(.+)\s*$", line)
            if m:
                key = m.group(1).strip().strip("'\"")
                break
    if not key:
        raise SystemExit("TIINGO_API_KEY not found (env or .env)")
    return key


def fetch(ticker: str, key: str, start: str = "1990-01-01") -> pd.DataFrame | None:
    """One symbol's full EOD history; None when Tiingo does not have it (404).
    Network errors back off and retry; a hard failure skips the ticker."""
    for attempt in range(4):
        try:
            r = requests.get(BASE.format(t=ticker.lower()),
                             params={"startDate": start, "format": "json"},
                             headers={"Content-Type": "application/json",
                                      "Authorization": f"Token {key}"}, timeout=60)
        except requests.RequestException as exc:
            print(f"{ticker}: {type(exc).__name__}; retry {attempt + 1}", flush=True)
            time.sleep(20 * (attempt + 1))
            continue
        if r.status_code == 429:
            print(f"{ticker}: rate limited; waiting 15 min", flush=True)
            time.sleep(900)
            continue
        if r.status_code in (402, 403):     # monthly symbol quota exhausted
            raise SystemExit(f"{ticker}: HTTP {r.status_code} — quota reached; "
                             "remaining tickers stay uncached for the next run")
        if r.status_code == 404:
            return None
        r.raise_for_status()
        d = r.json()
        if not d:
            return None
        df = pd.DataFrame(d)
        df["date"] = pd.to_datetime(df["date"]).dt.tz_localize(None)
        keep = [c for c in KEEP if c in df.columns]
        return df[keep].set_index("date")
    raise RuntimeError(f"{ticker}: network retries exhausted")


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("tickers", nargs="*")
    ap.add_argument("--file", help="one ticker per line")
    ap.add_argument("--min-interval", type=float, default=75.0,
                    help="seconds between network fetches (free tier allows 50/hour)")
    ap.add_argument("--start", default="1990-01-01")
    args = ap.parse_args()
    tickers = [l.strip() for l in open(args.file) if l.strip()] if args.file else args.tickers
    if not tickers:
        raise SystemExit("usage: fetch_tiingo TICKER ... | --file list.txt")

    os.makedirs(DIR, exist_ok=True)
    key = api_key()
    todo = [t for t in tickers if not os.path.exists(os.path.join(DIR, f"{t.upper()}.csv"))
            and not os.path.exists(os.path.join(DIR, f"{t.upper()}.miss"))]
    print(f"{len(tickers)} tickers, {len(todo)} to fetch", flush=True)
    fetched = failed = 0
    last = 0.0
    for i, t in enumerate(todo):
        wait = args.min_interval - (time.time() - last)
        if wait > 0:
            time.sleep(wait)
        last = time.time()
        try:
            df = fetch(t, key, args.start)
        except RuntimeError:
            print(f"[{i + 1}/{len(todo)}] {t:8s} network fail; retried next run", flush=True)
            continue
        if df is None:
            failed += 1
            with open(os.path.join(DIR, f"{t.upper()}.miss"), "w") as fh:
                fh.write("404\n")
            print(f"[{i + 1}/{len(todo)}] {t:8s} not found", flush=True)
            continue
        df.to_csv(os.path.join(DIR, f"{t.upper()}.csv"))
        fetched += 1
        eta_min = (len(todo) - i - 1) * args.min_interval / 60
        print(f"[{i + 1}/{len(todo)}] {t:8s} {len(df)} rows "
              f"{df.index.min().date()}..{df.index.max().date()} (ETA {eta_min:.0f}m)",
              flush=True)
    print(f"done: {fetched} fetched, {failed} not found", flush=True)


if __name__ == "__main__":
    main()