Gahow Wang
ad569f1cea
Register us_momentum_qqq_trend: half overnight_recovery_top10, half a QQQ trend sleeve (strategies/qqq_trend.py: 200-day switch times a 30% vol target, held as TQQQ + SHY). Mixed stock+ETF, US only (extra tickers QQQ, SHY, TQQQ). Research (research/lab/): alpha against QQQ with a 2006-16 design / 2017-26 holdout split (alpha.py, qqq_scan.py, beat_qqq.py); leverage and drawdown frontier (levered_book.py, us_target50.py); synthetic 3x ETFs from 1990 and TrendRider/QQQ-trend stress tests (fetch_long_history.py, letf_history.py, vol_trend.py); short-horizon sleeves, diversifiers, industry momentum, earnings-announcement returns; new point-in-time data sources: SEC Form 4 insider trades (fetch_insider.py, insider.py), 10-K/10-Q text similarity, 8-K earnings dates and SIC codes (fetch_lazy_prices.py, lazy_prices.py), and an S&P 400/600 universe from iShares holdings (fetch_us_mid.py, panel_mid.py). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
108 lines
4.0 KiB
Python
108 lines
4.0 KiB
Python
"""
|
|
Long daily history for the leveraged-ETF and cross-asset research
|
|
(research/lab/letf_history.py): stock indices from 1927/1985, the ETFs the
|
|
trend strategies trade, gold futures, bitcoin, and Treasury yields.
|
|
|
|
PYTHONPATH=. uv run python -m research.lab.fetch_long_history [--no-proxy]
|
|
|
|
Yahoo is fetched ticker by ticker (batched requests fail intermittently)
|
|
and each ticker is cached in data/research/ext/yahoo/ as it arrives, so a
|
|
re-run after rate limiting fetches only what is missing. The fetch stops
|
|
after three tickers fail in a row: Yahoo is rate limiting, and the live
|
|
monitor needs it too. FRED yields come from the public fredgraph CSV
|
|
endpoint. Writes data/research/long_history.pkl: {"close", "open"}
|
|
DataFrames (adjusted for splits and dividends) and "fred" (yields in
|
|
percent, and the Nasdaq-100 level).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import io
|
|
import os
|
|
import pickle
|
|
import time
|
|
import urllib.request
|
|
import warnings
|
|
|
|
import pandas as pd
|
|
import yfinance as yf
|
|
|
|
from research.lab.fetch_us import _proxy_env
|
|
|
|
warnings.filterwarnings("ignore")
|
|
OUT = os.path.join("data", "research", "long_history.pkl")
|
|
YAHOO = [
|
|
"^GSPC", "^NDX", "^VIX", # indices (price only)
|
|
"SPY", "QQQ", "IWM", "EFA", "EEM", "VNQ", "TLT", "IEF", "SHY", "GLD", "DBC", "GSG", "UUP",
|
|
"TQQQ", "UPRO", "QLD", "SSO", "SOXL", "TMF", "UGL", # leveraged ETFs (for validation)
|
|
"VFINX", "VUSTX", # mutual funds with dividends, 1980s-
|
|
"GC=F", "BTC-USD", "ETH-USD", "IBIT",
|
|
]
|
|
FRED = ["DTB3", "DGS2", "DGS10", "DGS30", "NASDAQ100"]
|
|
|
|
|
|
CACHE = os.path.join("data", "research", "ext", "yahoo")
|
|
|
|
|
|
def _cached(t: str) -> str:
|
|
return os.path.join(CACHE, t.replace("^", "_").replace("=", "_") + ".csv")
|
|
|
|
|
|
def fetch_yahoo(tickers: list[str], start: str = "1927-01-01") -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
os.makedirs(CACHE, exist_ok=True)
|
|
fails = 0
|
|
for t in tickers:
|
|
if os.path.exists(_cached(t)) or fails >= 3:
|
|
continue
|
|
for attempt in range(2):
|
|
try:
|
|
h = yf.Ticker(t).history(start=start, auto_adjust=True)
|
|
if len(h):
|
|
h.index = h.index.tz_localize(None).normalize()
|
|
h[["Open", "Close"]].to_csv(_cached(t))
|
|
break
|
|
except Exception as exc: # noqa: BLE001 — retry transient Yahoo errors
|
|
print(f" {t}: {type(exc).__name__}", flush=True)
|
|
time.sleep(20)
|
|
fails = 0 if os.path.exists(_cached(t)) else fails + 1
|
|
print(f"{t:8s} {'ok' if os.path.exists(_cached(t)) else 'MISSING'}", flush=True)
|
|
time.sleep(2)
|
|
close, opens = {}, {}
|
|
for t in tickers:
|
|
if os.path.exists(_cached(t)):
|
|
h = pd.read_csv(_cached(t), index_col=0, parse_dates=True)
|
|
close[t], opens[t] = h["Close"], h["Open"]
|
|
missing = [t for t in tickers if t not in close]
|
|
if missing:
|
|
print("missing (re-run later):", missing)
|
|
return pd.DataFrame(close).sort_index(), pd.DataFrame(opens).sort_index()
|
|
|
|
|
|
def fetch_fred(series: list[str]) -> pd.DataFrame:
|
|
out = {}
|
|
for s in series:
|
|
url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={s}"
|
|
with urllib.request.urlopen(url, timeout=60) as fh:
|
|
df = pd.read_csv(io.BytesIO(fh.read()), index_col=0, parse_dates=True)
|
|
out[s] = pd.to_numeric(df.iloc[:, 0], errors="coerce")
|
|
return pd.DataFrame(out)
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--no-proxy", action="store_true",
|
|
help="connect directly (when the proxy's IP is rate limited by Yahoo)")
|
|
if not ap.parse_args().no_proxy:
|
|
_proxy_env()
|
|
close, opens = fetch_yahoo(YAHOO)
|
|
fred = fetch_fred(FRED)
|
|
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
|
with open(OUT, "wb") as fh:
|
|
pickle.dump({"close": close, "open": opens, "fred": fred}, fh)
|
|
print(f"Saved {OUT}: {close.shape[1]} series, FRED {list(fred.columns)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|