""" Point-in-time insider-trading factors for the US panel, from officers' and directors' open-market Form 4 purchases (P) and sales (S) (research/lab/fetch_insider.py, SEC Insider Transactions Data Sets, 2006 →). A trade is usable from the first trading day strictly after its filing date, so a factor on date t only counts filings dated before t (the engine then trades at the open of t+1). A window of W calendar days counts a filing on t when t - W <= filed < t. Factors (dates x panel tickers). NaN where no issuer CIK covers the ticker on that date (``covered``), 0 where it is covered but nothing was filed: insider_buyers_90d distinct insiders with a purchase filed in the last 90 days minus distinct insiders with a sale insider_net_value_180d (purchase $ - sale $) filed in the last 180 days / market cap; NaN where fundamentals' mcap is missing (all names before 2009, ~10% after) insider_opportunistic opportunistic buyers minus opportunistic sellers, last 90 days (Cohen, Malloy & Pomorski 2012); NaN before 2009-04-01 (needs three prior years of trades) insider_cluster_buy 1 if >= 2 distinct officers/directors bought in the last 30 days, else 0 Routine vs opportunistic (CMP 2012, "Decoding Inside Information", p. 11-12): an insider who traded in each of the three preceding calendar years is "routine" for year Y if some calendar month m had a trade in each of Y-1, Y-2 and Y-3, else "opportunistic"; all her trades in Y get that label. Insiders without a trade in each of the three years are unclassified. The paper does not condition on direction; ``same_direction=True`` is the variant that counts only same-direction trades for the routine test. Classification is per (insider, issuer) and, stricter than the paper, only uses history filed before the classified trade's filing date. from research.lab import insider I = insider.load(p) # cached PYTHONPATH=. uv run python -m research.lab.insider # evaluation """ from __future__ import annotations import json import os import pickle import time import warnings import numpy as np import pandas as pd from research.lab import fundamentals as fu from research.lab.factor_scan import RESULTS from research.lab.factor_scan import load as load_panel from research.lab.report import stats warnings.filterwarnings("ignore") TRADES = os.path.join("data", "research", "insider_trades.pkl") CACHE = os.path.join("data", "research", "insider_factors.pkl") OPP_START = "2009-04-01" # first 90-day window made only of classifiable (2009+) trades COVER_SLACK = 120 # days after an issuer's last Form 4 that still count as covered NAMES = ["insider_buyers_90d", "insider_net_value_180d", "insider_opportunistic", "insider_cluster_buy"] def _raw() -> tuple[pd.DataFrame, pd.DataFrame]: with open(TRADES, "rb") as fh: raw = pickle.load(fh) return raw["trades"], raw["tickers"] # --------------------------------------------------------------------------- # trades → events # --------------------------------------------------------------------------- def classify_cmp(tr: pd.DataFrame, same_direction: bool = False) -> pd.Series: """'routine' / 'opportunistic' / None for every trade (index of ``tr``).""" d = tr[["insider_cik", "cik", "code", "trans_date", "filed"]].copy() d["y"] = d["trans_date"].dt.year d["m"] = d["trans_date"].dt.month ok = d["y"].notna() d = d[ok].astype({"y": int, "m": int}) d["fd"] = (d["filed"] - pd.Timestamp("1970-01-01")).dt.days.astype(float) # NaN compares False who = ["insider_cik", "cik"] mkey = who + (["code"] if same_direction else []) # first filing day of any trade per year, and of a trade per (year, month) any_y = d.groupby(who + ["y"])["fd"].min() mon = d.groupby(mkey + ["y", "m"])["fd"].min().unstack("m").reindex(columns=range(1, 13)) f = d["fd"].values hist = np.ones(len(d), bool) routine = np.ones((len(d), 12), bool) for k in (1, 2, 3): a = any_y.reindex(pd.MultiIndex.from_arrays([d[c] for c in who] + [d["y"] - k])).values hist &= a < f M = mon.reindex(pd.MultiIndex.from_arrays([d[c] for c in mkey] + [d["y"] - k])).values routine &= M < f[:, None] lab = np.where(hist & routine.any(axis=1), "routine", np.where(hist, "opportunistic", None)) return pd.Series(lab, index=d.index).reindex(tr.index) def _explode(tr: pd.DataFrame, tm: pd.DataFrame) -> pd.DataFrame: """One row per (trade, panel ticker): every share class of an issuer CIK (GOOG/GOOGL, FOX/FOXA, ...) gets that issuer's trades.""" alias = tm[tm["source"] == "sec_map"].groupby("cik")["ticker"].apply(sorted) multi = alias[alias.str.len() > 1] parts = [tr[~tr["cik"].isin(multi.index)]] for c, ts in multi.items(): sub = tr[tr["cik"] == c] parts += [sub.assign(ticker=t) for t in ts] return pd.concat(parts, ignore_index=True) def share_class_group(tm: pd.DataFrame) -> dict[str, str]: alias = tm[tm["source"] == "sec_map"].groupby("cik")["ticker"].apply(sorted) return {t: ts[0] for ts in alias for t in ts} def covered(tm: pd.DataFrame, dates: pd.DatetimeIndex, tickers: list[str]) -> pd.DataFrame: """True where an issuer CIK mapped to the ticker was filing Form 4s: from the session after its first filing in the data to COVER_SLACK days after its last one.""" col = {t: i for i, t in enumerate(tickers)} out = np.zeros((len(dates), len(tickers)), bool) for r in tm.itertuples(index=False): if r.ticker in col: a = dates.searchsorted(r.first, side="right") b = dates.searchsorted(r.last + pd.Timedelta(days=COVER_SLACK), side="right") out[a:b, col[r.ticker]] = True return pd.DataFrame(out, index=dates, columns=tickers) def window(ev: pd.DataFrame, dates: pd.DatetimeIndex, n_cols: int, days: int, weights: np.ndarray | None = None) -> np.ndarray: """(dates x columns) count of distinct ``insider_cik`` with a filing in the last ``days`` calendar days before each date; with ``weights``, the sum of the weights of those filings instead.""" filed = ev["filed"].values s = dates.searchsorted(filed, side="right") # first session after filing e = dates.searchsorted(filed + np.timedelta64(days, "D"), side="right") D = np.zeros((len(dates) + 1, n_cols)) c = ev["col"].values if weights is not None: np.add.at(D, (s, c), weights) np.add.at(D, (e, c), -weights) out = np.cumsum(D, axis=0)[:-1] out[np.abs(out) < 1.0] = 0.0 # float residue of the running sum return out # merge each insider's overlapping [s, e) spans so she counts once df = pd.DataFrame({"c": c, "k": ev["insider_cik"].values, "f": filed, "s": s, "e": e}) df = df.sort_values(["c", "k", "f"]) new = ((df["c"] != df["c"].shift()) | (df["k"] != df["k"].shift()) | (df["s"] > df["e"].shift())).values run = np.cumsum(new) g = df.groupby(run).agg(c=("c", "first"), s=("s", "min"), e=("e", "max")) np.add.at(D, (g["s"].values, g["c"].values), 1.0) np.add.at(D, (g["e"].values, g["c"].values), -1.0) return np.cumsum(D, axis=0)[:-1] # --------------------------------------------------------------------------- # build # --------------------------------------------------------------------------- def build(p) -> dict[str, pd.DataFrame]: t0 = time.time() tr, tm = _raw() dates, tickers = p.dates, list(p.close.columns) tr = tr.assign(cmp=classify_cmp(tr)) ex = _explode(tr, tm) col = pd.Series(np.arange(len(tickers)), index=tickers) ex = ex[ex["ticker"].isin(col.index)].copy() ex["col"] = col.reindex(ex["ticker"]).values cov = covered(tm, dates, tickers) n = len(tickers) buy, sell = ex[ex["code"] == "P"], ex[ex["code"] == "S"] print(f" {len(ex):,} trades on {ex['ticker'].nunique()} tickers [{time.time() - t0:.0f}s]", flush=True) def frame(x): return pd.DataFrame(x, index=dates, columns=tickers).where(cov).astype("float32") F = {} F["insider_buyers_90d"] = frame(window(buy, dates, n, 90) - window(sell, dates, n, 90)) signed = np.where(ex["code"] == "P", 1.0, -1.0) * ex["value"].fillna(0.0).values net = window(ex, dates, n, 180, weights=signed) mcap = fu.load(p)["mcap"].reindex(index=dates, columns=tickers) F["insider_net_value_180d"] = frame(net / mcap.values) ob = buy[buy["cmp"] == "opportunistic"] osl = sell[sell["cmp"] == "opportunistic"] opp = window(ob, dates, n, 90) - window(osl, dates, n, 90) opp[dates < pd.Timestamp(OPP_START)] = np.nan F["insider_opportunistic"] = frame(opp) F["insider_cluster_buy"] = frame((window(buy, dates, n, 30) >= 2).astype(float)) print(f" factors built [{time.time() - t0:.0f}s]", flush=True) return F def load(p, rebuild: bool = False) -> dict[str, pd.DataFrame]: if os.path.exists(CACHE) and not rebuild: with open(CACHE, "rb") as fh: return pickle.load(fh) F = build(p) with open(CACHE, "wb") as fh: pickle.dump(F, fh, protocol=4) return F # --------------------------------------------------------------------------- # evaluation # --------------------------------------------------------------------------- END = "2026-09-24" # 2010-2016 = the first period without the 2008-09 crash and rebound PERIODS = {"2006-2016": ("2006-01-01", "2016-12-31"), "2010-2016": ("2010-01-01", "2016-12-31"), "2017-2026": ("2017-01-01", END), "all": ("2006-01-01", END)} HORIZONS = (5, 21, 63) # first date each portfolio is evaluated: full 180-day window / classifiable # history / market caps for >= ~80% of members START = {"insider_buyers_90d": "2006-07-03", "insider_cluster_buy": "2006-07-03", "insider_opportunistic": OPP_START, "insider_net_value_180d": "2010-07-01"} LONG_START = "2006-07-03" def _avail_rows(dates: pd.DatetimeIndex, filed: pd.Series) -> np.ndarray: return dates.searchsorted(filed.values, side="right") def trade_events(p, ex: pd.DataFrame, group: dict[str, str]) -> pd.DataFrame: """Ticker-days on which a filing became usable, for PIT members only; share classes of one issuer count once.""" i = _avail_rows(p.dates, ex["filed"]) e = ex.assign(row=i)[i < len(p.dates)] e = e[p.member.values[e["row"].values, e["col"].values]] e = e.assign(grp=e["ticker"].map(lambda t: group.get(t, t))) return e.drop_duplicates(["grp", "row"])[["row", "col"]] def onset_events(p, flag: pd.DataFrame, group: dict[str, str]) -> pd.DataFrame: """Days a 0/1 panel switches from 0 to 1, for PIT members.""" v = flag.values prev = np.vstack([np.full((1, v.shape[1]), np.nan), v[:-1]]) on = (v == 1) & (prev == 0) & p.member.values r, c = np.nonzero(on) e = pd.DataFrame({"row": r, "col": c}) e["grp"] = [group.get(t, t) for t in flag.columns[c]] return e.drop_duplicates(["grp", "row"])[["row", "col"]] def excess_fwd(p, h: int) -> np.ndarray: """Open(t) → open(t+h) return minus the equal-weight PIT-member average.""" o = p.open.where(p.open > 0).ffill() f = o.shift(-h) / o - 1 ew = f.where(p.member).mean(axis=1) return f.sub(ew, axis=0).values def clustered_t(x: np.ndarray, month: np.ndarray) -> tuple[float, float, int]: ok = ~np.isnan(x) x, month = x[ok], month[ok] n = len(x) if n < 10: return float("nan"), float("nan"), n m = x.mean() g = pd.Series(x - m).groupby(month).sum() G = len(g) se = np.sqrt((g ** 2).sum() * G / max(G - 1, 1)) / n return float(m), float(m / se), n def event_study(p, events: dict[str, pd.DataFrame]) -> dict: X = {h: excess_fwd(p, h) for h in HORIZONS} ym = np.asarray(p.dates.year * 100 + p.dates.month) out = {} for kind, e in events.items(): out[kind] = {} for per, (a, b) in PERIODS.items(): ra, rb = p.dates.searchsorted(pd.Timestamp(a)), p.dates.searchsorted(pd.Timestamp(b), side="right") ee = e[(e["row"] >= ra) & (e["row"] < rb)] out[kind][per] = {} for h in HORIZONS: x = X[h][ee["row"].values, ee["col"].values] m, t, n = clustered_t(x, ym[ee["row"].values]) out[kind][per][f"+{h}"] = {"mean": m, "t": t, "n": n} return out def coverage(p, I: dict, ex: pd.DataFrame) -> pd.DataFrame: yr = p.dates.year mem = p.member cov = I["insider_buyers_90d"].notna() & mem i = _avail_rows(p.dates, ex["filed"]) e = ex.assign(row=i)[i < len(p.dates)] e = e[mem.values[e["row"].values, e["col"].values]] e = e.assign(year=p.dates[e["row"].values].year).drop_duplicates(["accession", "code"]) return pd.DataFrame({ "members": mem.sum(axis=1).groupby(yr).mean().round(0), "mapped": cov.sum(axis=1).groupby(yr).mean().round(0), "buy_filings": e[e["code"] == "P"].groupby("year").size(), "sell_filings": e[e["code"] == "S"].groupby("year").size(), "opp_buy_filings": e[(e["code"] == "P") & (e["cmp"] == "opportunistic")].groupby("year").size(), "names_bought": e[e["code"] == "P"].groupby("year")["ticker"].nunique(), }).loc[2006:].fillna(0).astype(int) def _active(r: pd.Series, bench: pd.Series) -> tuple[float, float]: """Annualized mean daily return over ``bench`` and its t-stat.""" act = (r - bench.reindex(r.index)).dropna() ir = act.mean() / act.std() * np.sqrt(252) return act.mean() * 252, ir * np.sqrt(len(act) / 252) def _period_stats(r: pd.Series, ew: pd.Series, tie: pd.Series, a: str, b: str) -> dict: r = r.loc[a:b].dropna() s = stats((1 + r).cumprod()) act, t = _active(r, ew) act_tie, t_tie = _active(r, tie) return {"sharpe": s["sharpe"], "cagr": s["cagr"], "mdd": s["mdd"], "active_ann": act, "active_t": t, "vs_tiebreak_ann": act_tie, "vs_tiebreak_t": t_tie} def main() -> None: from research.lab.rotation import factor_ranks from research.lab.us_sweep import _blend, run p, _ = load_panel("us") I = load(p) tr, tm = _raw() tr = tr.assign(cmp=classify_cmp(tr), cmp_dir=classify_cmp(tr, same_direction=True)) ex = _explode(tr, tm) col = pd.Series(np.arange(len(p.close.columns)), index=p.close.columns) ex = ex[ex["ticker"].isin(col.index)].copy() ex["col"] = col.reindex(ex["ticker"]).values group = share_class_group(tm) out: dict = {} # ---- mapping and coverage src = tm.groupby("source")["ticker"].nunique().to_dict() unmapped = sorted(set(p.close.columns) - set(tm["ticker"])) unm_mem = [t for t in unmapped if p.member[t].any()] print(f"ticker map: {src} ; unmapped ever-members: {unm_mem}") print("by symbol:", sorted(tm.loc[tm["source"] == "symbol", "ticker"])) cv = coverage(p, I, ex) print("\n=== coverage among PIT S&P 500 members (per year; filings = Form 4s with a P or S line)") print(cv.to_string()) out["mapping"] = {"by_source": src, "unmapped_ever_members": unm_mem, "by_symbol": sorted(tm.loc[tm["source"] == "symbol", "ticker"])} out["coverage"] = {int(k): {c: int(v) for c, v in r.items()} for k, r in cv.iterrows()} # ---- event study B, S = ex["code"] == "P", ex["code"] == "S" ev = { "all purchases": trade_events(p, ex[B], group), "opportunistic purchases": trade_events(p, ex[B & (ex["cmp"] == "opportunistic")], group), "cluster buys (onset)": onset_events(p, I["insider_cluster_buy"], group), "opportunistic sales": trade_events(p, ex[S & (ex["cmp"] == "opportunistic")], group), # references "routine purchases": trade_events(p, ex[B & (ex["cmp"] == "routine")], group), "all sales": trade_events(p, ex[S], group), "opp. purchases, same-direction rule": trade_events(p, ex[B & (ex["cmp_dir"] == "opportunistic")], group), "opp. sales, same-direction rule": trade_events(p, ex[S & (ex["cmp_dir"] == "opportunistic")], group), } es = event_study(p, ev) out["event_study"] = es print("\n=== event study: excess return vs EW members from the next open after filing, % " "(t clustered by month) [n]") print(f"{'':38s}" + "".join(f"{per:>27s}" for per in PERIODS)) for kind, d in es.items(): for h in HORIZONS: cells = "".join(f" {d[per][f'+{h}']['mean']*100:+6.2f} ({d[per][f'+{h}']['t']:+5.1f}) " f"[{d[per][f'+{h}']['n']:5d}]" for per in PERIODS) print(f"{(kind if h == 5 else ''):34s} +{h:<3d}{cells}") # ---- portfolios: top 20 among members, factor first, 12-1 momentum breaks ties pr = factor_ranks(p, "us", ["mom_12_1", "overnight_mom", "rec_126"]) tie = pr["mom_12_1"].fillna(0.0) def lexi(f): r = f.where(p.member).rank(axis=1, method="dense") return (r + 0.5 * tie).where(r.notna()) kw = dict(top_n=20, buffer=2.0, every=21) spans = lambda a: [(a, "2016-12-30"), ("2017-01-03", END)] # noqa: E731 R, runs = {}, {} ew_sc = p.close.notna().astype(float).where(p.member) runs["ew_universe"] = run(p, ew_sc, spans(LONG_START), top_n=10000, every=21) runs["mom_12_1 alone (tie-breaker)"] = run(p, pr["mom_12_1"].where(p.member), spans(LONG_START), **kw) mom = _blend(pr["overnight_mom"], pr["rec_126"]) runs["momentum sleeve"] = run(p, mom.where(p.member), spans(LONG_START), top_n=10, buffer=2.0, every=10) for name in NAMES: runs[name] = run(p, lexi(I[name]), spans(START[name]), **kw) print(f" ran {name}", flush=True) for k, v in runs.items(): R[k] = v["_rets"] ew, ms, tb = R["ew_universe"], R["momentum sleeve"], R["mom_12_1 alone (tie-breaker)"] print("\n=== long-only top 20 (every 21 days, buffer 2, next-open fills, 10 bp); " "periods: start-2016 | 2017-2026 | whole") print(" sharpe / cagr / mdd per period; active = vs EW universe, vs-tie = vs the 12-1 momentum " "tie-breaker alone (ann. %, t); phase-avg block Sharpe; corr with momentum sleeve (raw, active vs EW)") port = {} for k, v in runs.items(): start = START.get(k, LONG_START) pers = {"p1": (start, "2016-12-30"), "p2": ("2017-01-03", END), "all": (start, END)} st = {pk: _period_stats(R[k], ew, tb, a, b) for pk, (a, b) in pers.items()} r = R[k].loc[start:END] c_raw = r.corr(ms.loc[start:END]) c_act = (r - ew.loc[start:END]).corr(ms.loc[start:END] - ew.loc[start:END]) n_sig = None if k in I: f = I[k].where(p.member).loc[start:END] n_sig = float((f > 0).sum(axis=1).mean()) port[k] = {"start": start, **st, "blocks": v["blocks"], "phase_sharpe": v["sharpe"], "phase_cagr": v["cagr"], "phase_mdd": v["mdd"], "turn": v["turn"], "corr_mom": c_raw, "corr_mom_active": c_act, "avg_signal_names": n_sig} cells = " | ".join(f"{st[pk]['sharpe']:.2f} {st[pk]['cagr']*100:5.1f}% {st[pk]['mdd']*100:5.1f}%" for pk in ("p1", "p2", "all")) act = " | ".join(f"{st[pk]['active_ann']*100:+5.1f}% ({st[pk]['active_t']:+4.1f})" for pk in ("p1", "p2", "all")) vst = " | ".join(f"{st[pk]['vs_tiebreak_ann']*100:+5.1f}% ({st[pk]['vs_tiebreak_t']:+4.1f})" for pk in ("p1", "p2", "all")) sig = f" sig-names {n_sig:5.1f}" if n_sig is not None else "" print(f"{k:30s} from {start} {cells}\n{'':30s} active {act} blocks " f"[{' '.join(f'{b:.2f}' for b in v['blocks'])}] corr {c_raw:.2f}/{c_act:+.2f} " f"turn {v['turn']:.1f}x{sig}") if k in I: print(f"{'':30s} vs-tie {vst}") out["portfolio"] = port os.makedirs(RESULTS, exist_ok=True) path = os.path.join(RESULTS, "insider_eval.json") with open(path, "w") as fh: json.dump(out, fh, indent=1, default=float) print(f"\nSaved {path}") if __name__ == "__main__": main()