"""GDELT news-volume factor scan on the S&P 500 PIT panel.

Factors from the daily news-intensity series (share of monitored news):
  news_spike = volume / 60-day rolling median
  news_chg5  = 5-day mean / 21-day mean - 1
  news_z     = z-score of the 5-day mean vs its 252-day history

PIT: GDELT's day-D series is final at D's end; signals are lagged 1 session
(usable at the close of D+1 at the earliest, matching the engine convention).

    PYTHONPATH=. uv run python -m research.lab.news_volume
"""
from __future__ import annotations

import json
import os
import pickle

import numpy as np
import pandas as pd

GDELT_PATH = "data/research/gdelt_volume.pkl"


def factors(vol: pd.DataFrame) -> dict[str, pd.DataFrame]:
    med60 = vol.rolling(60, min_periods=20).median()
    f = {}
    f["news_spike"] = vol / med60.replace(0.0, np.nan)
    m5 = vol.rolling(5, min_periods=3).mean()
    m21 = vol.rolling(21, min_periods=10).mean()
    f["news_chg5"] = m5 / m21.replace(0.0, np.nan) - 1.0
    f["news_z"] = (m5 - m5.rolling(252, min_periods=60).mean()) / m5.rolling(
        252, min_periods=60).std().replace(0.0, np.nan)
    return f


def main() -> None:
    with open("data/research/us.panel.pkl", "rb") as fh:
        p = pickle.load(fh)
    with open(GDELT_PATH, "rb") as fh:
        vol = pickle.load(fh)["volume"].reindex(columns=p.close.columns)
    print(f"gdelt panel: {vol.shape}, tickers with data: {int(vol.notna().any().sum())}")
    close = p.close
    member = p.member.reindex_like(close).fillna(False)
    fwd1 = close.shift(-1) / close - 1.0
    fwd21 = close.shift(-21) / close - 1.0
    months = fwd1.index.to_period("M")
    rows = []
    for fname, fraw in factors(vol).items():
        f = fraw.shift(1).reindex(index=close.index).where(member)
        ics1, ics21 = [], []
        for per in sorted(set(months)):
            idx = months == per
            xdf = f.loc[idx]
            if xdf.notna().sum(axis=1).max() < 50:
                continue
            for ydf, out in ((fwd1.loc[idx], ics1), (fwd21.loc[idx], ics21)):
                mask = xdf.notna().to_numpy() & np.isfinite(ydf.reindex(columns=xdf.columns).to_numpy())
                if mask.sum() < 100:
                    continue
                xr = xdf.rank(axis=1).to_numpy()[mask]
                yr = ydf.reindex(columns=xdf.columns).where(mask).rank(axis=1).to_numpy()[mask]
                xm, ym = xr - xr.mean(), yr - yr.mean()
                denom = np.sqrt((xm ** 2).sum() * (ym ** 2).sum())
                if denom > 0:
                    out.append(float((xm * ym).sum() / denom))
        ic1, ic21 = np.array(ics1), np.array(ics21)
        rows.append({"factor": fname,
                     "IC1": ic1.mean(), "t1": ic1.mean() / ic1.std() * np.sqrt(len(ic1)) if len(ic1) > 1 else np.nan,
                     "IC21": ic21.mean(), "t21": ic21.mean() / ic21.std() * np.sqrt(len(ic21)) if len(ic21) > 1 else np.nan,
                     "months": len(ic21)})
    df = pd.DataFrame(rows)
    print(df.round(4).to_string(index=False))
    df.to_csv("data/research/results/news_volume_scan.csv", index=False)


if __name__ == "__main__":
    main()