research: commit the 2026-10-07/08 studies left in the working tree, overnight mid/small (STOP)
Studies run by earlier sessions and documented in CLAUDE.md, but never committed: FINRA short volume (SVR) and the research-only svr_momentum veto, SEC fails-to-deliver, 13D activists and the 13D event sleeve, 13F institutional flow, GDELT news volume, S&P 400/600 Form 4 insiders, union_shield, shield_lab; plus the mid/small insider and Tiingo fetcher changes and midsmall_rebuild. overnight_midsmall: the overnight-only momentum book on S&P 400/600 earns 0.93 net Sharpe at 3 bp a side (2019-26) with corr 0.83 to the S&P 500 book (STOP). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
123
research/lab/fetch_gdelt.py
Normal file
123
research/lab/fetch_gdelt.py
Normal file
@@ -0,0 +1,123 @@
|
||||
"""Fetch GDELT news-volume timelines for the S&P 500 panel.
|
||||
|
||||
One GDELT DOC 2.0 timelinevol query per company returns the full daily
|
||||
series since 2017 (news volume intensity = share of all monitored news).
|
||||
Company names come from the SEC ticker map (company_tickers.json); legal
|
||||
suffixes are trimmed for the phrase query. The public API is burst-limited
|
||||
(429s), so requests are paced with adaptive backoff.
|
||||
|
||||
Cache: data/research/gdelt/<ticker>.json (resume-safe). Output panel:
|
||||
data/research/gdelt_volume.pkl {"volume": DataFrame} on daily dates.
|
||||
|
||||
PYTHONPATH=. uv run python -m research.lab.fetch_gdelt [--limit N]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from research.lab.fetch_us import _proxy_env
|
||||
|
||||
OUT_DIR = os.path.join("data", "research", "gdelt")
|
||||
PANEL = os.path.join("data", "research", "gdelt_volume.pkl")
|
||||
TICKERS_CACHE = "data/research/company_tickers.json"
|
||||
UA = {"User-Agent": "quant-research/0.1 gahow@example.com"}
|
||||
BASE = ("https://api.gdeltproject.org/api/v2/doc/doc?query={q}%20sourcelang:english"
|
||||
"&mode=timelinevol&format=json&startdatetime=20170101000000&enddatetime=20261007000000")
|
||||
SUFFIX = re.compile(r",?\s*(inc|incorporated|corp|corporation|co|company|ltd|limited|plc|"
|
||||
r"holdings?|group|the|nv|sa|ag|llc|lp|trust|reit)\.?$", re.I)
|
||||
|
||||
|
||||
def clean_name(title: str) -> str:
|
||||
name = title.strip().rstrip(".")
|
||||
prev = None
|
||||
while prev != name:
|
||||
prev = name
|
||||
name = SUFFIX.sub("", name).strip().rstrip(",")
|
||||
return name
|
||||
|
||||
|
||||
def query(name: str) -> list | None:
|
||||
q = requests.utils.quote(f'"{name}"')
|
||||
url = BASE.format(q=q)
|
||||
delay = 5.0
|
||||
for attempt in range(5):
|
||||
try:
|
||||
r = requests.get(url, headers=UA, timeout=90)
|
||||
if r.status_code == 429:
|
||||
time.sleep(delay)
|
||||
delay = min(delay * 2, 60)
|
||||
continue
|
||||
r.raise_for_status()
|
||||
d = r.json()
|
||||
series = d.get("timeline", [{}])[0].get("data", [])
|
||||
return series
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if attempt == 4:
|
||||
print(f" FAILED {name}: {exc}")
|
||||
else:
|
||||
time.sleep(delay)
|
||||
delay = min(delay * 2, 60)
|
||||
return None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
_proxy_env()
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
limit = None
|
||||
if "--limit" in sys.argv:
|
||||
limit = int(sys.argv[sys.argv.index("--limit") + 1])
|
||||
close = pd.read_csv("data/us.csv", index_col=0, nrows=0)
|
||||
tickers = list(close.columns)
|
||||
raw = json.load(open(TICKERS_CACHE))
|
||||
title_by_ticker = {}
|
||||
for v in raw.values():
|
||||
t = str(v["ticker"]).upper().replace(".", "-")
|
||||
title_by_ticker.setdefault(t, str(v["title"]))
|
||||
todo = [t for t in tickers if t in title_by_ticker and
|
||||
not os.path.exists(os.path.join(OUT_DIR, f"{t}.json"))]
|
||||
if limit:
|
||||
todo = todo[:limit]
|
||||
print(f"tickers: {len(tickers)}, cached: {len(tickers)-len(todo)}, fetching: {len(todo)}")
|
||||
for i, t in enumerate(todo):
|
||||
series = query(clean_name(title_by_ticker[t]))
|
||||
if series is None:
|
||||
continue
|
||||
with open(os.path.join(OUT_DIR, f"{t}.json"), "w") as fh:
|
||||
json.dump(series, fh)
|
||||
if (i + 1) % 25 == 0:
|
||||
print(f" {i+1}/{len(todo)} (last {t}, {len(series)} points)", flush=True)
|
||||
time.sleep(12.0)
|
||||
|
||||
# assemble panel
|
||||
frames = {}
|
||||
for t in tickers:
|
||||
path = os.path.join(OUT_DIR, f"{t}.json")
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
try:
|
||||
series = json.load(open(path))
|
||||
idx = pd.to_datetime([p["date"][:8] for p in series], format="%Y%m%d")
|
||||
vals = [p["value"] for p in series]
|
||||
frames[t] = pd.Series(vals, index=idx)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
if frames:
|
||||
df = pd.DataFrame(frames).sort_index()
|
||||
df = df[~df.index.duplicated(keep="last")]
|
||||
with open(PANEL, "wb") as fh:
|
||||
pickle.dump({"volume": df.astype("float32")}, fh)
|
||||
print(f"panel {df.shape} -> {PANEL}")
|
||||
else:
|
||||
print("no cached series yet")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user