quant-trading/research/lab/insider.py

"""
Point-in-time insider-trading factors for the US panel, from officers' and
directors' open-market Form 4 purchases (P) and sales (S)
(research/lab/fetch_insider.py, SEC Insider Transactions Data Sets, 2006 →).

A trade is usable from the first trading day strictly after its filing date,
so a factor on date t only counts filings dated before t (the engine then
trades at the open of t+1). A window of W calendar days counts a filing on t
when t - W <= filed < t.

Factors (dates x panel tickers). NaN where no issuer CIK covers the ticker on
that date (``covered``), 0 where it is covered but nothing was filed:
  insider_buyers_90d      distinct insiders with a purchase filed in the last
                          90 days minus distinct insiders with a sale
  insider_net_value_180d  (purchase $ - sale $) filed in the last 180 days /
                          market cap; NaN where fundamentals' mcap is missing
                          (all names before 2009, ~10% after)
  insider_opportunistic   opportunistic buyers minus opportunistic sellers,
                          last 90 days (Cohen, Malloy & Pomorski 2012); NaN
                          before 2009-04-01 (needs three prior years of trades)
  insider_cluster_buy     1 if >= 2 distinct officers/directors bought in the
                          last 30 days, else 0

Routine vs opportunistic (CMP 2012, "Decoding Inside Information", p. 11-12):
an insider who traded in each of the three preceding calendar years is
"routine" for year Y if some calendar month m had a trade in each of Y-1, Y-2
and Y-3, else "opportunistic"; all her trades in Y get that label. Insiders
without a trade in each of the three years are unclassified. The paper does
not condition on direction; ``same_direction=True`` is the variant that
counts only same-direction trades for the routine test. Classification is per
(insider, issuer) and, stricter than the paper, only uses history filed
before the classified trade's filing date.

    from research.lab import insider
    I = insider.load(p)                                   # cached
    PYTHONPATH=. uv run python -m research.lab.insider    # evaluation
"""

from __future__ import annotations

import json
import os
import pickle
import time
import warnings

import numpy as np
import pandas as pd

from research.lab import fundamentals as fu
from research.lab.factor_scan import RESULTS
from research.lab.factor_scan import load as load_panel
from research.lab.report import stats

warnings.filterwarnings("ignore")

TRADES = os.path.join("data", "research", "insider_trades.pkl")
CACHE = os.path.join("data", "research", "insider_factors.pkl")
OPP_START = "2009-04-01"      # first 90-day window made only of classifiable (2009+) trades
COVER_SLACK = 120             # days after an issuer's last Form 4 that still count as covered
NAMES = ["insider_buyers_90d", "insider_net_value_180d", "insider_opportunistic", "insider_cluster_buy"]


def _raw() -> tuple[pd.DataFrame, pd.DataFrame]:
    with open(TRADES, "rb") as fh:
        raw = pickle.load(fh)
    return raw["trades"], raw["tickers"]


# ---------------------------------------------------------------------------
# trades → events
# ---------------------------------------------------------------------------

def classify_cmp(tr: pd.DataFrame, same_direction: bool = False) -> pd.Series:
    """'routine' / 'opportunistic' / None for every trade (index of ``tr``)."""
    d = tr[["insider_cik", "cik", "code", "trans_date", "filed"]].copy()
    d["y"] = d["trans_date"].dt.year
    d["m"] = d["trans_date"].dt.month
    ok = d["y"].notna()
    d = d[ok].astype({"y": int, "m": int})
    d["fd"] = (d["filed"] - pd.Timestamp("1970-01-01")).dt.days.astype(float)   # NaN compares False
    who = ["insider_cik", "cik"]
    mkey = who + (["code"] if same_direction else [])
    # first filing day of any trade per year, and of a trade per (year, month)
    any_y = d.groupby(who + ["y"])["fd"].min()
    mon = d.groupby(mkey + ["y", "m"])["fd"].min().unstack("m").reindex(columns=range(1, 13))
    f = d["fd"].values
    hist = np.ones(len(d), bool)
    routine = np.ones((len(d), 12), bool)
    for k in (1, 2, 3):
        a = any_y.reindex(pd.MultiIndex.from_arrays([d[c] for c in who] + [d["y"] - k])).values
        hist &= a < f
        M = mon.reindex(pd.MultiIndex.from_arrays([d[c] for c in mkey] + [d["y"] - k])).values
        routine &= M < f[:, None]
    lab = np.where(hist & routine.any(axis=1), "routine", np.where(hist, "opportunistic", None))
    return pd.Series(lab, index=d.index).reindex(tr.index)


def _explode(tr: pd.DataFrame, tm: pd.DataFrame) -> pd.DataFrame:
    """One row per (trade, panel ticker): every share class of an issuer CIK
    (GOOG/GOOGL, FOX/FOXA, ...) gets that issuer's trades."""
    alias = tm[tm["source"] == "sec_map"].groupby("cik")["ticker"].apply(sorted)
    multi = alias[alias.str.len() > 1]
    parts = [tr[~tr["cik"].isin(multi.index)]]
    for c, ts in multi.items():
        sub = tr[tr["cik"] == c]
        parts += [sub.assign(ticker=t) for t in ts]
    return pd.concat(parts, ignore_index=True)


def share_class_group(tm: pd.DataFrame) -> dict[str, str]:
    alias = tm[tm["source"] == "sec_map"].groupby("cik")["ticker"].apply(sorted)
    return {t: ts[0] for ts in alias for t in ts}


def covered(tm: pd.DataFrame, dates: pd.DatetimeIndex, tickers: list[str]) -> pd.DataFrame:
    """True where an issuer CIK mapped to the ticker was filing Form 4s: from
    the session after its first filing in the data to COVER_SLACK days after
    its last one."""
    col = {t: i for i, t in enumerate(tickers)}
    out = np.zeros((len(dates), len(tickers)), bool)
    for r in tm.itertuples(index=False):
        if r.ticker in col:
            a = dates.searchsorted(r.first, side="right")
            b = dates.searchsorted(r.last + pd.Timedelta(days=COVER_SLACK), side="right")
            out[a:b, col[r.ticker]] = True
    return pd.DataFrame(out, index=dates, columns=tickers)


def window(ev: pd.DataFrame, dates: pd.DatetimeIndex, n_cols: int, days: int,
           weights: np.ndarray | None = None) -> np.ndarray:
    """(dates x columns) count of distinct ``insider_cik`` with a filing in the
    last ``days`` calendar days before each date; with ``weights``, the sum of
    the weights of those filings instead."""
    filed = ev["filed"].values
    s = dates.searchsorted(filed, side="right")                  # first session after filing
    e = dates.searchsorted(filed + np.timedelta64(days, "D"), side="right")
    D = np.zeros((len(dates) + 1, n_cols))
    c = ev["col"].values
    if weights is not None:
        np.add.at(D, (s, c), weights)
        np.add.at(D, (e, c), -weights)
        out = np.cumsum(D, axis=0)[:-1]
        out[np.abs(out) < 1.0] = 0.0                              # float residue of the running sum
        return out
    # merge each insider's overlapping [s, e) spans so she counts once
    df = pd.DataFrame({"c": c, "k": ev["insider_cik"].values, "f": filed, "s": s, "e": e})
    df = df.sort_values(["c", "k", "f"])
    new = ((df["c"] != df["c"].shift()) | (df["k"] != df["k"].shift())
           | (df["s"] > df["e"].shift())).values
    run = np.cumsum(new)
    g = df.groupby(run).agg(c=("c", "first"), s=("s", "min"), e=("e", "max"))
    np.add.at(D, (g["s"].values, g["c"].values), 1.0)
    np.add.at(D, (g["e"].values, g["c"].values), -1.0)
    return np.cumsum(D, axis=0)[:-1]


# ---------------------------------------------------------------------------
# build
# ---------------------------------------------------------------------------

def build(p) -> dict[str, pd.DataFrame]:
    t0 = time.time()
    tr, tm = _raw()
    dates, tickers = p.dates, list(p.close.columns)
    tr = tr.assign(cmp=classify_cmp(tr))
    ex = _explode(tr, tm)
    col = pd.Series(np.arange(len(tickers)), index=tickers)
    ex = ex[ex["ticker"].isin(col.index)].copy()
    ex["col"] = col.reindex(ex["ticker"]).values
    cov = covered(tm, dates, tickers)
    n = len(tickers)
    buy, sell = ex[ex["code"] == "P"], ex[ex["code"] == "S"]
    print(f"  {len(ex):,} trades on {ex['ticker'].nunique()} tickers [{time.time() - t0:.0f}s]", flush=True)

    def frame(x):
        return pd.DataFrame(x, index=dates, columns=tickers).where(cov).astype("float32")

    F = {}
    F["insider_buyers_90d"] = frame(window(buy, dates, n, 90) - window(sell, dates, n, 90))
    signed = np.where(ex["code"] == "P", 1.0, -1.0) * ex["value"].fillna(0.0).values
    net = window(ex, dates, n, 180, weights=signed)
    mcap = fu.load(p)["mcap"].reindex(index=dates, columns=tickers)
    F["insider_net_value_180d"] = frame(net / mcap.values)
    ob = buy[buy["cmp"] == "opportunistic"]
    osl = sell[sell["cmp"] == "opportunistic"]
    opp = window(ob, dates, n, 90) - window(osl, dates, n, 90)
    opp[dates < pd.Timestamp(OPP_START)] = np.nan
    F["insider_opportunistic"] = frame(opp)
    F["insider_cluster_buy"] = frame((window(buy, dates, n, 30) >= 2).astype(float))
    print(f"  factors built [{time.time() - t0:.0f}s]", flush=True)
    return F


def load(p, rebuild: bool = False) -> dict[str, pd.DataFrame]:
    if os.path.exists(CACHE) and not rebuild:
        with open(CACHE, "rb") as fh:
            return pickle.load(fh)
    F = build(p)
    with open(CACHE, "wb") as fh:
        pickle.dump(F, fh, protocol=4)
    return F


# ---------------------------------------------------------------------------
# evaluation
# ---------------------------------------------------------------------------

END = "2026-09-24"
# 2010-2016 = the first period without the 2008-09 crash and rebound
PERIODS = {"2006-2016": ("2006-01-01", "2016-12-31"), "2010-2016": ("2010-01-01", "2016-12-31"),
           "2017-2026": ("2017-01-01", END), "all": ("2006-01-01", END)}
HORIZONS = (5, 21, 63)
# first date each portfolio is evaluated: full 180-day window / classifiable
# history / market caps for >= ~80% of members
START = {"insider_buyers_90d": "2006-07-03", "insider_cluster_buy": "2006-07-03",
         "insider_opportunistic": OPP_START, "insider_net_value_180d": "2010-07-01"}
LONG_START = "2006-07-03"


def _avail_rows(dates: pd.DatetimeIndex, filed: pd.Series) -> np.ndarray:
    return dates.searchsorted(filed.values, side="right")


def trade_events(p, ex: pd.DataFrame, group: dict[str, str]) -> pd.DataFrame:
    """Ticker-days on which a filing became usable, for PIT members only;
    share classes of one issuer count once."""
    i = _avail_rows(p.dates, ex["filed"])
    e = ex.assign(row=i)[i < len(p.dates)]
    e = e[p.member.values[e["row"].values, e["col"].values]]
    e = e.assign(grp=e["ticker"].map(lambda t: group.get(t, t)))
    return e.drop_duplicates(["grp", "row"])[["row", "col"]]


def onset_events(p, flag: pd.DataFrame, group: dict[str, str]) -> pd.DataFrame:
    """Days a 0/1 panel switches from 0 to 1, for PIT members."""
    v = flag.values
    prev = np.vstack([np.full((1, v.shape[1]), np.nan), v[:-1]])
    on = (v == 1) & (prev == 0) & p.member.values
    r, c = np.nonzero(on)
    e = pd.DataFrame({"row": r, "col": c})
    e["grp"] = [group.get(t, t) for t in flag.columns[c]]
    return e.drop_duplicates(["grp", "row"])[["row", "col"]]


def excess_fwd(p, h: int) -> np.ndarray:
    """Open(t) → open(t+h) return minus the equal-weight PIT-member average."""
    o = p.open.where(p.open > 0).ffill()
    f = o.shift(-h) / o - 1
    ew = f.where(p.member).mean(axis=1)
    return f.sub(ew, axis=0).values


def clustered_t(x: np.ndarray, month: np.ndarray) -> tuple[float, float, int]:
    ok = ~np.isnan(x)
    x, month = x[ok], month[ok]
    n = len(x)
    if n < 10:
        return float("nan"), float("nan"), n
    m = x.mean()
    g = pd.Series(x - m).groupby(month).sum()
    G = len(g)
    se = np.sqrt((g ** 2).sum() * G / max(G - 1, 1)) / n
    return float(m), float(m / se), n


def event_study(p, events: dict[str, pd.DataFrame]) -> dict:
    X = {h: excess_fwd(p, h) for h in HORIZONS}
    ym = np.asarray(p.dates.year * 100 + p.dates.month)
    out = {}
    for kind, e in events.items():
        out[kind] = {}
        for per, (a, b) in PERIODS.items():
            ra, rb = p.dates.searchsorted(pd.Timestamp(a)), p.dates.searchsorted(pd.Timestamp(b), side="right")
            ee = e[(e["row"] >= ra) & (e["row"] < rb)]
            out[kind][per] = {}
            for h in HORIZONS:
                x = X[h][ee["row"].values, ee["col"].values]
                m, t, n = clustered_t(x, ym[ee["row"].values])
                out[kind][per][f"+{h}"] = {"mean": m, "t": t, "n": n}
    return out


def coverage(p, I: dict, ex: pd.DataFrame) -> pd.DataFrame:
    yr = p.dates.year
    mem = p.member
    cov = I["insider_buyers_90d"].notna() & mem
    i = _avail_rows(p.dates, ex["filed"])
    e = ex.assign(row=i)[i < len(p.dates)]
    e = e[mem.values[e["row"].values, e["col"].values]]
    e = e.assign(year=p.dates[e["row"].values].year).drop_duplicates(["accession", "code"])
    return pd.DataFrame({
        "members": mem.sum(axis=1).groupby(yr).mean().round(0),
        "mapped": cov.sum(axis=1).groupby(yr).mean().round(0),
        "buy_filings": e[e["code"] == "P"].groupby("year").size(),
        "sell_filings": e[e["code"] == "S"].groupby("year").size(),
        "opp_buy_filings": e[(e["code"] == "P") & (e["cmp"] == "opportunistic")].groupby("year").size(),
        "names_bought": e[e["code"] == "P"].groupby("year")["ticker"].nunique(),
    }).loc[2006:].fillna(0).astype(int)


def _active(r: pd.Series, bench: pd.Series) -> tuple[float, float]:
    """Annualized mean daily return over ``bench`` and its t-stat."""
    act = (r - bench.reindex(r.index)).dropna()
    ir = act.mean() / act.std() * np.sqrt(252)
    return act.mean() * 252, ir * np.sqrt(len(act) / 252)


def _period_stats(r: pd.Series, ew: pd.Series, tie: pd.Series, a: str, b: str) -> dict:
    r = r.loc[a:b].dropna()
    s = stats((1 + r).cumprod())
    act, t = _active(r, ew)
    act_tie, t_tie = _active(r, tie)
    return {"sharpe": s["sharpe"], "cagr": s["cagr"], "mdd": s["mdd"],
            "active_ann": act, "active_t": t, "vs_tiebreak_ann": act_tie, "vs_tiebreak_t": t_tie}


def main() -> None:
    from research.lab.rotation import factor_ranks
    from research.lab.us_sweep import _blend, run

    p, _ = load_panel("us")
    I = load(p)
    tr, tm = _raw()
    tr = tr.assign(cmp=classify_cmp(tr), cmp_dir=classify_cmp(tr, same_direction=True))
    ex = _explode(tr, tm)
    col = pd.Series(np.arange(len(p.close.columns)), index=p.close.columns)
    ex = ex[ex["ticker"].isin(col.index)].copy()
    ex["col"] = col.reindex(ex["ticker"]).values
    group = share_class_group(tm)
    out: dict = {}

    # ---- mapping and coverage
    src = tm.groupby("source")["ticker"].nunique().to_dict()
    unmapped = sorted(set(p.close.columns) - set(tm["ticker"]))
    unm_mem = [t for t in unmapped if p.member[t].any()]
    print(f"ticker map: {src} ; unmapped ever-members: {unm_mem}")
    print("by symbol:", sorted(tm.loc[tm["source"] == "symbol", "ticker"]))
    cv = coverage(p, I, ex)
    print("\n=== coverage among PIT S&P 500 members (per year; filings = Form 4s with a P or S line)")
    print(cv.to_string())
    out["mapping"] = {"by_source": src, "unmapped_ever_members": unm_mem,
                      "by_symbol": sorted(tm.loc[tm["source"] == "symbol", "ticker"])}
    out["coverage"] = {int(k): {c: int(v) for c, v in r.items()} for k, r in cv.iterrows()}

    # ---- event study
    B, S = ex["code"] == "P", ex["code"] == "S"
    ev = {
        "all purchases": trade_events(p, ex[B], group),
        "opportunistic purchases": trade_events(p, ex[B & (ex["cmp"] == "opportunistic")], group),
        "cluster buys (onset)": onset_events(p, I["insider_cluster_buy"], group),
        "opportunistic sales": trade_events(p, ex[S & (ex["cmp"] == "opportunistic")], group),
        # references
        "routine purchases": trade_events(p, ex[B & (ex["cmp"] == "routine")], group),
        "all sales": trade_events(p, ex[S], group),
        "opp. purchases, same-direction rule": trade_events(p, ex[B & (ex["cmp_dir"] == "opportunistic")], group),
        "opp. sales, same-direction rule": trade_events(p, ex[S & (ex["cmp_dir"] == "opportunistic")], group),
    }
    es = event_study(p, ev)
    out["event_study"] = es
    print("\n=== event study: excess return vs EW members from the next open after filing, % "
          "(t clustered by month) [n]")
    print(f"{'':38s}" + "".join(f"{per:>27s}" for per in PERIODS))
    for kind, d in es.items():
        for h in HORIZONS:
            cells = "".join(f"  {d[per][f'+{h}']['mean']*100:+6.2f} ({d[per][f'+{h}']['t']:+5.1f}) "
                            f"[{d[per][f'+{h}']['n']:5d}]" for per in PERIODS)
            print(f"{(kind if h == 5 else ''):34s} +{h:<3d}{cells}")

    # ---- portfolios: top 20 among members, factor first, 12-1 momentum breaks ties
    pr = factor_ranks(p, "us", ["mom_12_1", "overnight_mom", "rec_126"])
    tie = pr["mom_12_1"].fillna(0.0)

    def lexi(f):
        r = f.where(p.member).rank(axis=1, method="dense")
        return (r + 0.5 * tie).where(r.notna())

    kw = dict(top_n=20, buffer=2.0, every=21)
    spans = lambda a: [(a, "2016-12-30"), ("2017-01-03", END)]   # noqa: E731
    R, runs = {}, {}
    ew_sc = p.close.notna().astype(float).where(p.member)
    runs["ew_universe"] = run(p, ew_sc, spans(LONG_START), top_n=10000, every=21)
    runs["mom_12_1 alone (tie-breaker)"] = run(p, pr["mom_12_1"].where(p.member), spans(LONG_START), **kw)
    mom = _blend(pr["overnight_mom"], pr["rec_126"])
    runs["momentum sleeve"] = run(p, mom.where(p.member), spans(LONG_START), top_n=10, buffer=2.0, every=10)
    for name in NAMES:
        runs[name] = run(p, lexi(I[name]), spans(START[name]), **kw)
        print(f"  ran {name}", flush=True)
    for k, v in runs.items():
        R[k] = v["_rets"]
    ew, ms, tb = R["ew_universe"], R["momentum sleeve"], R["mom_12_1 alone (tie-breaker)"]

    print("\n=== long-only top 20 (every 21 days, buffer 2, next-open fills, 10 bp); "
          "periods: start-2016 | 2017-2026 | whole")
    print("    sharpe / cagr / mdd per period; active = vs EW universe, vs-tie = vs the 12-1 momentum "
          "tie-breaker alone (ann. %, t); phase-avg block Sharpe; corr with momentum sleeve (raw, active vs EW)")
    port = {}
    for k, v in runs.items():
        start = START.get(k, LONG_START)
        pers = {"p1": (start, "2016-12-30"), "p2": ("2017-01-03", END), "all": (start, END)}
        st = {pk: _period_stats(R[k], ew, tb, a, b) for pk, (a, b) in pers.items()}
        r = R[k].loc[start:END]
        c_raw = r.corr(ms.loc[start:END])
        c_act = (r - ew.loc[start:END]).corr(ms.loc[start:END] - ew.loc[start:END])
        n_sig = None
        if k in I:
            f = I[k].where(p.member).loc[start:END]
            n_sig = float((f > 0).sum(axis=1).mean())
        port[k] = {"start": start, **st, "blocks": v["blocks"], "phase_sharpe": v["sharpe"],
                   "phase_cagr": v["cagr"], "phase_mdd": v["mdd"], "turn": v["turn"],
                   "corr_mom": c_raw, "corr_mom_active": c_act, "avg_signal_names": n_sig}
        cells = " | ".join(f"{st[pk]['sharpe']:.2f} {st[pk]['cagr']*100:5.1f}% {st[pk]['mdd']*100:5.1f}%"
                           for pk in ("p1", "p2", "all"))
        act = " | ".join(f"{st[pk]['active_ann']*100:+5.1f}% ({st[pk]['active_t']:+4.1f})" for pk in ("p1", "p2", "all"))
        vst = " | ".join(f"{st[pk]['vs_tiebreak_ann']*100:+5.1f}% ({st[pk]['vs_tiebreak_t']:+4.1f})"
                         for pk in ("p1", "p2", "all"))
        sig = f" sig-names {n_sig:5.1f}" if n_sig is not None else ""
        print(f"{k:30s} from {start}  {cells}\n{'':30s} active {act}  blocks "
              f"[{' '.join(f'{b:.2f}' for b in v['blocks'])}]  corr {c_raw:.2f}/{c_act:+.2f}  "
              f"turn {v['turn']:.1f}x{sig}")
        if k in I:
            print(f"{'':30s} vs-tie {vst}")
    out["portfolio"] = port
    os.makedirs(RESULTS, exist_ok=True)
    path = os.path.join(RESULTS, "insider_eval.json")
    with open(path, "w") as fh:
        json.dump(out, fh, indent=1, default=float)
    print(f"\nSaved {path}")


if __name__ == "__main__":
    main()