Gahow Wang
0e378efa67
Event-driven long-only simulator with next-open fills, CN tradability (halts, limit-up/down, stamp duty), PIT S&P 500 / CSI 300 membership, rebalance-phase averaging, factor IC scans, maximin stepwise composite search across 5-year blocks, walk-forward LightGBM/ridge combiners, overlays, and deflated Sharpe. Fetchers pull 20y US OHLCV from Yahoo and CSI 300/500 ever-members (membership snapshot from baostock, bars from Yahoo). Research data lives in data/research/ (ignored). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
44 lines
1.5 KiB
Python
44 lines
1.5 KiB
Python
"""Fetch GICS-like sectors from Yahoo for S&P 500 ever-members missing from
|
|
data/us_sectors.csv. Writes data/research/us_sectors_extra.csv.
|
|
|
|
uv run python -m research.lab.fetch_us_sectors
|
|
"""
|
|
|
|
import os
|
|
import pickle
|
|
|
|
import pandas as pd
|
|
import yfinance as yf
|
|
|
|
from research.lab.fetch_us import _proxy_env
|
|
|
|
# Yahoo sector names → GICS sector names used in data/us_sectors.csv
|
|
YAHOO_TO_GICS = {
|
|
"Technology": "Information Technology", "Healthcare": "Health Care",
|
|
"Financial Services": "Financials", "Consumer Cyclical": "Consumer Discretionary",
|
|
"Consumer Defensive": "Consumer Staples", "Basic Materials": "Materials",
|
|
"Communication Services": "Communication Services", "Industrials": "Industrials",
|
|
"Energy": "Energy", "Utilities": "Utilities", "Real Estate": "Real Estate",
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
_proxy_env()
|
|
with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh:
|
|
tickers = list(pickle.load(fh)["close"].columns)
|
|
known = set(pd.read_csv(os.path.join("data", "us_sectors.csv"))["ticker"])
|
|
rows = []
|
|
for t in sorted(set(tickers) - known):
|
|
try:
|
|
s = yf.Ticker(t).info.get("sector")
|
|
except Exception: # noqa: BLE001
|
|
s = None
|
|
rows.append({"ticker": t, "sector": YAHOO_TO_GICS.get(s, s)})
|
|
out = pd.DataFrame(rows)
|
|
out.to_csv(os.path.join("data", "research", "us_sectors_extra.csv"), index=False)
|
|
print(out["sector"].value_counts(dropna=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|