Gahow Wang
04e2c5c3be
Studies run by earlier sessions and documented in CLAUDE.md, but never committed: FINRA short volume (SVR) and the research-only svr_momentum veto, SEC fails-to-deliver, 13D activists and the 13D event sleeve, 13F institutional flow, GDELT news volume, S&P 400/600 Form 4 insiders, union_shield, shield_lab; plus the mid/small insider and Tiingo fetcher changes and midsmall_rebuild. overnight_midsmall: the overnight-only momentum book on S&P 400/600 earns 0.93 net Sharpe at 3 bp a side (2019-26) with corr 0.83 to the S&P 500 book (STOP). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
79 lines
3.1 KiB
Python
79 lines
3.1 KiB
Python
"""GDELT news-volume factor scan on the S&P 500 PIT panel.
|
|
|
|
Factors from the daily news-intensity series (share of monitored news):
|
|
news_spike = volume / 60-day rolling median
|
|
news_chg5 = 5-day mean / 21-day mean - 1
|
|
news_z = z-score of the 5-day mean vs its 252-day history
|
|
|
|
PIT: GDELT's day-D series is final at D's end; signals are lagged 1 session
|
|
(usable at the close of D+1 at the earliest, matching the engine convention).
|
|
|
|
PYTHONPATH=. uv run python -m research.lab.news_volume
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import pickle
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
GDELT_PATH = "data/research/gdelt_volume.pkl"
|
|
|
|
|
|
def factors(vol: pd.DataFrame) -> dict[str, pd.DataFrame]:
|
|
med60 = vol.rolling(60, min_periods=20).median()
|
|
f = {}
|
|
f["news_spike"] = vol / med60.replace(0.0, np.nan)
|
|
m5 = vol.rolling(5, min_periods=3).mean()
|
|
m21 = vol.rolling(21, min_periods=10).mean()
|
|
f["news_chg5"] = m5 / m21.replace(0.0, np.nan) - 1.0
|
|
f["news_z"] = (m5 - m5.rolling(252, min_periods=60).mean()) / m5.rolling(
|
|
252, min_periods=60).std().replace(0.0, np.nan)
|
|
return f
|
|
|
|
|
|
def main() -> None:
|
|
with open("data/research/us.panel.pkl", "rb") as fh:
|
|
p = pickle.load(fh)
|
|
with open(GDELT_PATH, "rb") as fh:
|
|
vol = pickle.load(fh)["volume"].reindex(columns=p.close.columns)
|
|
print(f"gdelt panel: {vol.shape}, tickers with data: {int(vol.notna().any().sum())}")
|
|
close = p.close
|
|
member = p.member.reindex_like(close).fillna(False)
|
|
fwd1 = close.shift(-1) / close - 1.0
|
|
fwd21 = close.shift(-21) / close - 1.0
|
|
months = fwd1.index.to_period("M")
|
|
rows = []
|
|
for fname, fraw in factors(vol).items():
|
|
f = fraw.shift(1).reindex(index=close.index).where(member)
|
|
ics1, ics21 = [], []
|
|
for per in sorted(set(months)):
|
|
idx = months == per
|
|
xdf = f.loc[idx]
|
|
if xdf.notna().sum(axis=1).max() < 50:
|
|
continue
|
|
for ydf, out in ((fwd1.loc[idx], ics1), (fwd21.loc[idx], ics21)):
|
|
mask = xdf.notna().to_numpy() & np.isfinite(ydf.reindex(columns=xdf.columns).to_numpy())
|
|
if mask.sum() < 100:
|
|
continue
|
|
xr = xdf.rank(axis=1).to_numpy()[mask]
|
|
yr = ydf.reindex(columns=xdf.columns).where(mask).rank(axis=1).to_numpy()[mask]
|
|
xm, ym = xr - xr.mean(), yr - yr.mean()
|
|
denom = np.sqrt((xm ** 2).sum() * (ym ** 2).sum())
|
|
if denom > 0:
|
|
out.append(float((xm * ym).sum() / denom))
|
|
ic1, ic21 = np.array(ics1), np.array(ics21)
|
|
rows.append({"factor": fname,
|
|
"IC1": ic1.mean(), "t1": ic1.mean() / ic1.std() * np.sqrt(len(ic1)) if len(ic1) > 1 else np.nan,
|
|
"IC21": ic21.mean(), "t21": ic21.mean() / ic21.std() * np.sqrt(len(ic21)) if len(ic21) > 1 else np.nan,
|
|
"months": len(ic21)})
|
|
df = pd.DataFrame(rows)
|
|
print(df.round(4).to_string(index=False))
|
|
df.to_csv("data/research/results/news_volume_scan.csv", index=False)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|