feat: us_momentum_qqq_trend; alpha-vs-QQQ research across new data sources
Register us_momentum_qqq_trend: half overnight_recovery_top10, half a QQQ trend sleeve (strategies/qqq_trend.py: 200-day switch times a 30% vol target, held as TQQQ + SHY). Mixed stock+ETF, US only (extra tickers QQQ, SHY, TQQQ). Research (research/lab/): alpha against QQQ with a 2006-16 design / 2017-26 holdout split (alpha.py, qqq_scan.py, beat_qqq.py); leverage and drawdown frontier (levered_book.py, us_target50.py); synthetic 3x ETFs from 1990 and TrendRider/QQQ-trend stress tests (fetch_long_history.py, letf_history.py, vol_trend.py); short-horizon sleeves, diversifiers, industry momentum, earnings-announcement returns; new point-in-time data sources: SEC Form 4 insider trades (fetch_insider.py, insider.py), 10-K/10-Q text similarity, 8-K earnings dates and SIC codes (fetch_lazy_prices.py, lazy_prices.py), and an S&P 400/600 universe from iShares holdings (fetch_us_mid.py, panel_mid.py). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
107
research/lab/fetch_long_history.py
Normal file
107
research/lab/fetch_long_history.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Long daily history for the leveraged-ETF and cross-asset research
|
||||
(research/lab/letf_history.py): stock indices from 1927/1985, the ETFs the
|
||||
trend strategies trade, gold futures, bitcoin, and Treasury yields.
|
||||
|
||||
PYTHONPATH=. uv run python -m research.lab.fetch_long_history [--no-proxy]
|
||||
|
||||
Yahoo is fetched ticker by ticker (batched requests fail intermittently)
|
||||
and each ticker is cached in data/research/ext/yahoo/ as it arrives, so a
|
||||
re-run after rate limiting fetches only what is missing. The fetch stops
|
||||
after three tickers fail in a row: Yahoo is rate limiting, and the live
|
||||
monitor needs it too. FRED yields come from the public fredgraph CSV
|
||||
endpoint. Writes data/research/long_history.pkl: {"close", "open"}
|
||||
DataFrames (adjusted for splits and dividends) and "fred" (yields in
|
||||
percent, and the Nasdaq-100 level).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import os
|
||||
import pickle
|
||||
import time
|
||||
import urllib.request
|
||||
import warnings
|
||||
|
||||
import pandas as pd
|
||||
import yfinance as yf
|
||||
|
||||
from research.lab.fetch_us import _proxy_env
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
OUT = os.path.join("data", "research", "long_history.pkl")
|
||||
YAHOO = [
|
||||
"^GSPC", "^NDX", "^VIX", # indices (price only)
|
||||
"SPY", "QQQ", "IWM", "EFA", "EEM", "VNQ", "TLT", "IEF", "SHY", "GLD", "DBC", "GSG", "UUP",
|
||||
"TQQQ", "UPRO", "QLD", "SSO", "SOXL", "TMF", "UGL", # leveraged ETFs (for validation)
|
||||
"VFINX", "VUSTX", # mutual funds with dividends, 1980s-
|
||||
"GC=F", "BTC-USD", "ETH-USD", "IBIT",
|
||||
]
|
||||
FRED = ["DTB3", "DGS2", "DGS10", "DGS30", "NASDAQ100"]
|
||||
|
||||
|
||||
CACHE = os.path.join("data", "research", "ext", "yahoo")
|
||||
|
||||
|
||||
def _cached(t: str) -> str:
|
||||
return os.path.join(CACHE, t.replace("^", "_").replace("=", "_") + ".csv")
|
||||
|
||||
|
||||
def fetch_yahoo(tickers: list[str], start: str = "1927-01-01") -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
os.makedirs(CACHE, exist_ok=True)
|
||||
fails = 0
|
||||
for t in tickers:
|
||||
if os.path.exists(_cached(t)) or fails >= 3:
|
||||
continue
|
||||
for attempt in range(2):
|
||||
try:
|
||||
h = yf.Ticker(t).history(start=start, auto_adjust=True)
|
||||
if len(h):
|
||||
h.index = h.index.tz_localize(None).normalize()
|
||||
h[["Open", "Close"]].to_csv(_cached(t))
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001 — retry transient Yahoo errors
|
||||
print(f" {t}: {type(exc).__name__}", flush=True)
|
||||
time.sleep(20)
|
||||
fails = 0 if os.path.exists(_cached(t)) else fails + 1
|
||||
print(f"{t:8s} {'ok' if os.path.exists(_cached(t)) else 'MISSING'}", flush=True)
|
||||
time.sleep(2)
|
||||
close, opens = {}, {}
|
||||
for t in tickers:
|
||||
if os.path.exists(_cached(t)):
|
||||
h = pd.read_csv(_cached(t), index_col=0, parse_dates=True)
|
||||
close[t], opens[t] = h["Close"], h["Open"]
|
||||
missing = [t for t in tickers if t not in close]
|
||||
if missing:
|
||||
print("missing (re-run later):", missing)
|
||||
return pd.DataFrame(close).sort_index(), pd.DataFrame(opens).sort_index()
|
||||
|
||||
|
||||
def fetch_fred(series: list[str]) -> pd.DataFrame:
|
||||
out = {}
|
||||
for s in series:
|
||||
url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={s}"
|
||||
with urllib.request.urlopen(url, timeout=60) as fh:
|
||||
df = pd.read_csv(io.BytesIO(fh.read()), index_col=0, parse_dates=True)
|
||||
out[s] = pd.to_numeric(df.iloc[:, 0], errors="coerce")
|
||||
return pd.DataFrame(out)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--no-proxy", action="store_true",
|
||||
help="connect directly (when the proxy's IP is rate limited by Yahoo)")
|
||||
if not ap.parse_args().no_proxy:
|
||||
_proxy_env()
|
||||
close, opens = fetch_yahoo(YAHOO)
|
||||
fred = fetch_fred(FRED)
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
with open(OUT, "wb") as fh:
|
||||
pickle.dump({"close": close, "open": opens, "fred": fred}, fh)
|
||||
print(f"Saved {OUT}: {close.shape[1]} series, FRED {list(fred.columns)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user