"""Fetch GICS-like sectors from Yahoo for S&P 500 ever-members missing from
data/us_sectors.csv. Writes data/research/us_sectors_extra.csv.

    uv run python -m research.lab.fetch_us_sectors
"""

import os
import pickle

import pandas as pd
import yfinance as yf

from research.lab.fetch_us import _proxy_env

# Yahoo sector names → GICS sector names used in data/us_sectors.csv
YAHOO_TO_GICS = {
    "Technology": "Information Technology", "Healthcare": "Health Care",
    "Financial Services": "Financials", "Consumer Cyclical": "Consumer Discretionary",
    "Consumer Defensive": "Consumer Staples", "Basic Materials": "Materials",
    "Communication Services": "Communication Services", "Industrials": "Industrials",
    "Energy": "Energy", "Utilities": "Utilities", "Real Estate": "Real Estate",
}


def main() -> None:
    _proxy_env()
    with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh:
        tickers = list(pickle.load(fh)["close"].columns)
    known = set(pd.read_csv(os.path.join("data", "us_sectors.csv"))["ticker"])
    rows = []
    for t in sorted(set(tickers) - known):
        try:
            s = yf.Ticker(t).info.get("sector")
        except Exception:  # noqa: BLE001
            s = None
        rows.append({"ticker": t, "sector": YAHOO_TO_GICS.get(s, s)})
    out = pd.DataFrame(rows)
    out.to_csv(os.path.join("data", "research", "us_sectors_extra.csv"), index=False)
    print(out["sector"].value_counts(dropna=False))


if __name__ == "__main__":
    main()