Gahow Wang
04e2c5c3be
Studies run by earlier sessions and documented in CLAUDE.md, but never committed: FINRA short volume (SVR) and the research-only svr_momentum veto, SEC fails-to-deliver, 13D activists and the 13D event sleeve, 13F institutional flow, GDELT news volume, S&P 400/600 Form 4 insiders, union_shield, shield_lab; plus the mid/small insider and Tiingo fetcher changes and midsmall_rebuild. overnight_midsmall: the overnight-only momentum book on S&P 400/600 earns 0.93 net Sharpe at 3 bp a side (2019-26) with corr 0.83 to the S&P 500 book (STOP). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
287 lines
14 KiB
Python
287 lines
14 KiB
Python
"""
|
|
Form 4 officer/director open-market purchase factors on the PIT S&P 400 and
|
|
S&P 600 panels (fetch_insider.py --universe mid|small, SEC quarterly data
|
|
sets 2006 ->; a trade is usable the session after its filing). The S&P 500
|
|
scan (research/lab/insider.py) found nothing; the literature places the
|
|
insider effect in the neglected small caps, so this is the mid/small test.
|
|
|
|
PYTHONPATH=. uv run python -m research.lab.insider_mid --universe mid
|
|
PYTHONPATH=. uv run python -m research.lab.insider_mid --universe small
|
|
|
|
Factors (same definitions as insider.py): insider_buyers_90d (net distinct
|
|
buyers, 90d), insider_net_value_180d (net $ / mcap, 180d), insider_
|
|
opportunistic (CMP 2012 classification, net distinct, 90d), insider_cluster_
|
|
buy (>= 2 distinct buyers, 30d). Sleeves use the midcap_sleeve construction
|
|
(top 30, equal weight, every 21 sessions, buffer 1, next-open fills, 20 bp a
|
|
side; 40 bp reported for S&P 600), averaged over the 21 rebalance phases.
|
|
The holdout hurdle is the EW PIT-member basket (which beat IJR by +2.8%/yr,
|
|
t 3.2, in the S&P 600 replication), not just the ETF.
|
|
|
|
Result (2026-10-09): all four factors REJECT in both universes. S&P 400
|
|
holdout 2017-26 (alpha vs the EW member basket): buyers -1.8%/yr (t -0.79),
|
|
net_value -1.9% (-0.79), opportunistic -0.1% (-0.06), cluster +0.3% (0.20;
|
|
halves +3.1/-2.8, the same 2017-21/2022-26 flip as the quality sleeves).
|
|
S&P 600: buyers -4.2%/yr (t -1.48 in both halves negative — insider buyers
|
|
cluster in broken names), net_value +0.7% (0.22, halves +5.2/-3.4),
|
|
opportunistic -1.6% (-0.86), cluster -1.6% (-0.89); at 40 bp all worse. The
|
|
pooled 400+600 event study finds a real but tiny drift after any purchase
|
|
(+5d excess +0.24%, t 2.8 in 2017-26; +63d +0.95%, t 1.8) that does not
|
|
survive the 20 bp round trip at portfolio scale, and the opportunistic (CMP)
|
|
split shows nothing (n ~1,000). Form 4 purchases do not extend the book into
|
|
mid/small caps — the same answer as the S&P 500 scan.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import dataclasses
|
|
import json
|
|
import os
|
|
import pickle
|
|
import warnings
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
from research.lab import insider as ins
|
|
from research.lab.alpha import alpha_stats
|
|
from research.lab.engine import Costs
|
|
from research.lab.factors import overnight_mom, rec_126
|
|
from research.lab.panel_mid import load_us_mid
|
|
from research.lab.us_final import monthly_blend
|
|
from research.lab.us_sweep import _blend, run
|
|
|
|
warnings.filterwarnings("ignore")
|
|
OUT = os.path.join("data", "research", "results", "insider_midsmall")
|
|
BOOK = os.path.join("data", "research", "results", "universe_midcap", "returns.pkl")
|
|
TRADES = {"mid": os.path.join("data", "research", "insider_trades_mid.pkl"),
|
|
"small": os.path.join("data", "research", "insider_trades_small.pkl")}
|
|
ETF = {"mid": "IJH", "small": "IJR"}
|
|
HOLDOUT = ("2017-01-03", "2026-09-24")
|
|
HALVES = [("2017-01-03", "2021-12-31"), ("2022-01-03", "2026-09-24")]
|
|
REFERENCE = ("2010-01-04", "2016-12-30")
|
|
HORIZONS = (5, 21, 63)
|
|
EV_PERIODS = {"2010-2016": ("2010-01-01", "2016-12-30"),
|
|
"2017-2026": ("2017-01-03", "2026-09-24"),
|
|
"2017-2021": ("2017-01-03", "2021-12-31"),
|
|
"2022-2026": ("2022-01-03", "2026-09-24")}
|
|
KW = dict(top_n=30, buffer=1.0, every=21)
|
|
PREREG = {
|
|
"question": "Do officer/director Form 4 open-market purchase factors earn alpha over the EW PIT-member "
|
|
"basket among S&P 400 / S&P 600 members?",
|
|
"why": "the S&P 500 scan (insider.py) showed nothing; the insider-purchase effect is classically strongest "
|
|
"in neglected small caps, which a small account can trade",
|
|
"arms": ["c0_momentum (control)", "insider_buyers_90d", "insider_net_value_180d",
|
|
"insider_opportunistic", "insider_cluster_buy"],
|
|
"construction": "top 30, equal weight, every 21 sessions, buffer 1, next-open fills, 20 bp a side "
|
|
"(40 bp reported for S&P 600), 21 phases averaged (same as midcap_sleeve.py)",
|
|
"holdout": "2017-01-03..2026-09-24; halves 2017-2021 / 2022-2026",
|
|
"accept_if_all": [
|
|
"A1: active return vs the EW PIT-member basket, holdout, Newey-West t >= 2.0, or t >= 1.5 with "
|
|
"active > 0 in both halves",
|
|
"A2: Sharpe(2/3 book + 1/3 sleeve) > Sharpe(2/3 book + 1/3 ETF), monthly reset, holdout "
|
|
"(book = universe_midcap base arm)",
|
|
],
|
|
"on_accept": "forward-only validation book; no production change",
|
|
"multiple_testing": "4 factors x 2 universes = 8 sleeves; a lone t just above 2 is weak evidence; report all",
|
|
"reference_only": f"{REFERENCE[0]}..{REFERENCE[1]}: price coverage 55-78%, survivor-biased",
|
|
"status": "holdout partly seen in earlier mid/small studies; Tiingo-extended panel build 2026-10-08; "
|
|
"not independent OOS",
|
|
}
|
|
|
|
|
|
def build_factors(p, uni: str, factors: dict | None = None) -> tuple[dict, pd.DataFrame, pd.DataFrame]:
|
|
"""(factor panels, exploded trades, ticker map) for one universe."""
|
|
with open(TRADES[uni], "rb") as fh:
|
|
raw = pickle.load(fh)
|
|
tr, tm = raw["trades"], raw["tickers"]
|
|
if factors is not None:
|
|
tr = tr.assign(cmp=factors)
|
|
else:
|
|
tr = tr.assign(cmp=ins.classify_cmp(tr))
|
|
ex = ins._explode(tr, tm)
|
|
col = pd.Series(np.arange(len(p.close.columns)), index=p.close.columns)
|
|
ex = ex[ex["ticker"].isin(col.index)].copy()
|
|
ex["col"] = col.reindex(ex["ticker"]).values
|
|
dates, tickers = p.dates, list(p.close.columns)
|
|
cov = ins.covered(tm, dates, tickers)
|
|
n = len(tickers)
|
|
buy, sell = ex[ex["code"] == "P"], ex[ex["code"] == "S"]
|
|
|
|
def frame(x):
|
|
return pd.DataFrame(x, index=dates, columns=tickers).where(cov).astype("float32")
|
|
|
|
from research.lab import fundamentals as fu
|
|
F = {}
|
|
F["insider_buyers_90d"] = frame(ins.window(buy, dates, n, 90) - ins.window(sell, dates, n, 90))
|
|
signed = np.where(ex["code"] == "P", 1.0, -1.0) * ex["value"].fillna(0.0).values
|
|
net = ins.window(ex, dates, n, 180, weights=signed)
|
|
mcap = fu.load_mid(p, which=uni)["mcap"].reindex(index=dates, columns=tickers)
|
|
F["insider_net_value_180d"] = frame(net / mcap.values)
|
|
ob, osl = buy[buy["cmp"] == "opportunistic"], sell[sell["cmp"] == "opportunistic"]
|
|
opp = ins.window(ob, dates, n, 90) - ins.window(osl, dates, n, 90)
|
|
opp[dates < pd.Timestamp(ins.OPP_START)] = np.nan
|
|
F["insider_opportunistic"] = frame(opp)
|
|
F["insider_cluster_buy"] = frame((ins.window(buy, dates, n, 30) >= 2).astype(float))
|
|
return F, ex, tm
|
|
|
|
|
|
def event_study(p, events: dict[str, pd.DataFrame]) -> dict:
|
|
"""Market-adjusted CARs with month-clustered t (mirrors insider.event_study
|
|
with mid/small periods)."""
|
|
X = {h: ins.excess_fwd(p, h) for h in HORIZONS}
|
|
ym = np.asarray(p.dates.year * 100 + p.dates.month)
|
|
out = {}
|
|
for kind, e in events.items():
|
|
out[kind] = {}
|
|
for per, (a, b) in EV_PERIODS.items():
|
|
ra = p.dates.searchsorted(pd.Timestamp(a))
|
|
rb = p.dates.searchsorted(pd.Timestamp(b), side="right")
|
|
ee = e[(e["row"] >= ra) & (e["row"] < rb)]
|
|
out[kind][per] = {}
|
|
for h in HORIZONS:
|
|
x = X[h][ee["row"].values, ee["col"].values]
|
|
m, t, n = ins.clustered_t(x, ym[ee["row"].values])
|
|
out[kind][per][f"+{h}"] = {"mean": m, "t": t, "n": n}
|
|
return out
|
|
|
|
|
|
def score(f: pd.DataFrame, eligible: pd.DataFrame) -> pd.DataFrame:
|
|
return _blend(f.where(eligible).rank(axis=1, pct=True)).where(eligible)
|
|
|
|
|
|
def sharpe(r: pd.Series) -> float:
|
|
return float(r.mean() / r.std() * np.sqrt(252))
|
|
|
|
|
|
def describe(r: pd.Series, ew: pd.Series, etf: pd.Series, book: pd.Series) -> dict:
|
|
"""Holdout stats; ``ew`` is the EW PIT-member basket (the hurdle), ``etf``
|
|
the universe ETF (reported)."""
|
|
h = r.loc[HOLDOUT[0]:HOLDOUT[1]]
|
|
e = (1 + h).cumprod()
|
|
a_ew, a_etf = alpha_stats(h, ew), alpha_stats(h, etf)
|
|
halves = [alpha_stats(r.loc[x:y], ew)["alpha"] for x, y in HALVES]
|
|
R = pd.DataFrame({"book": book, "s": r, "etf": etf}).loc[HOLDOUT[0]:HOLDOUT[1]].dropna()
|
|
with_s = monthly_blend(R, {"book": 2 / 3, "s": 1 / 3})
|
|
with_e = monthly_blend(R, {"book": 2 / 3, "etf": 1 / 3})
|
|
ref = r.loc[REFERENCE[0]:REFERENCE[1]]
|
|
return {"cagr": float(e.iloc[-1] ** (252 / len(h)) - 1), "sharpe": sharpe(h),
|
|
"mdd": float((e / e.cummax() - 1).min()),
|
|
"alpha_ew": float(a_ew["alpha"]), "alpha_ew_t": float(a_ew["t"]),
|
|
"alpha_etf": float(a_etf["alpha"]), "alpha_etf_t": float(a_etf["t"]),
|
|
"alpha_halves_vs_ew": [float(x) for x in halves],
|
|
"corr_book": float(R["s"].corr(R["book"])),
|
|
"book_sharpe_with_sleeve": sharpe(with_s), "book_sharpe_with_etf": sharpe(with_e),
|
|
"reference_sharpe": sharpe(ref),
|
|
"years": {int(k): float(v) for k, v in ((1 + h).groupby(h.index.year).prod() - 1).items()}}
|
|
|
|
|
|
def verdict(d: dict) -> dict:
|
|
a1 = d["alpha_ew_t"] >= 2.0 or (d["alpha_ew_t"] >= 1.5 and min(d["alpha_halves_vs_ew"]) > 0)
|
|
a2 = d["book_sharpe_with_sleeve"] > d["book_sharpe_with_etf"]
|
|
checks = {"A1": a1, "A2": a2}
|
|
return {"checks": {k: bool(v) for k, v in checks.items()},
|
|
"verdict": "ACCEPT" if all(checks.values()) else "REJECT"}
|
|
|
|
|
|
def run_universe(uni: str) -> dict:
|
|
out_dir = os.path.join(OUT, uni)
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
pre = os.path.join(out_dir, "prereg.json")
|
|
if not os.path.exists(pre):
|
|
with open(pre, "w") as fh:
|
|
json.dump({**PREREG, "universe": uni, "benchmark_hurdle": "EW members",
|
|
"etf": ETF[uni]}, fh, indent=2)
|
|
res_path = os.path.join(out_dir, "results.json")
|
|
done = json.load(open(res_path)) if os.path.exists(res_path) else {}
|
|
if done:
|
|
raise SystemExit(f"{res_path} exists; this study does not overwrite itself")
|
|
|
|
p = load_us_mid(uni)
|
|
bench = p.extra["etf"][ETF[uni]].pct_change(fill_method=None)
|
|
book = pd.read_pickle(BOOK)["base"]
|
|
F, ex, tm = build_factors(p, uni)
|
|
mem = p.member
|
|
S = {"c0_momentum": _blend(overnight_mom(p).where(mem).rank(axis=1, pct=True),
|
|
rec_126(p).where(mem).rank(axis=1, pct=True)).where(mem)}
|
|
for name in ("insider_buyers_90d", "insider_net_value_180d", "insider_opportunistic",
|
|
"insider_cluster_buy"):
|
|
S[name] = score(F[name], mem)
|
|
|
|
span = [(REFERENCE[0], HOLDOUT[1])]
|
|
costs = {"": p.costs} if uni == "mid" else {"": p.costs, "@40bp": Costs(buy=0.004, sell=0.004)}
|
|
rets = {}
|
|
for tag, c in costs.items():
|
|
pc = dataclasses.replace(p, costs=c)
|
|
for name, sc in S.items():
|
|
rets[name + tag] = run(pc, sc, span, **KW)["_rets"]
|
|
print(f"ran {uni} {name}{tag}", flush=True)
|
|
ew = p.close.pct_change(fill_method=None).where(p.member.shift(1, fill_value=False)).mean(axis=1)
|
|
rets["bench_ew_members"] = ew
|
|
rets[f"bench_{ETF[uni].lower()}"] = bench
|
|
|
|
for name, r in rets.items():
|
|
d = describe(r.reindex(p.dates).fillna(0.0), ew, bench, book)
|
|
if name.startswith("insider_") and not name.endswith("@40bp"):
|
|
d.update(verdict(d))
|
|
done[name] = d
|
|
if not name.startswith("bench"):
|
|
print(f"{name:26s} cagr {d['cagr']*100:5.1f}% sh {d['sharpe']:.2f} mdd {d['mdd']*100:6.1f}% "
|
|
f"a_EW {d['alpha_ew']*100:+5.1f}% t {d['alpha_ew_t']:+.2f} halves "
|
|
f"{[round(x*100, 1) for x in d['alpha_halves_vs_ew']]} a_ETF t {d['alpha_etf_t']:+.2f} "
|
|
f"book+sleeve {d['book_sharpe_with_sleeve']:.2f} vs book+ETF {d['book_sharpe_with_etf']:.2f} "
|
|
f"ref {d['reference_sharpe']:.2f} {d.get('verdict', '')} {d.get('checks', '')}", flush=True)
|
|
|
|
# ---- pooled event study on the midsmall panel
|
|
with open(TRADES["mid"], "rb") as fh:
|
|
raw_mid = pickle.load(fh)
|
|
with open(TRADES["small"], "rb") as fh:
|
|
raw_sm = pickle.load(fh)
|
|
pm = load_us_mid("midsmall")
|
|
tr = pd.concat([raw_mid["trades"], raw_sm["trades"]], ignore_index=True)
|
|
tr = tr.drop_duplicates(["accession", "code", "ticker", "trans_date", "shares", "price", "ownership"])
|
|
tr = tr.assign(cmp=ins.classify_cmp(tr))
|
|
tm2 = pd.concat([raw_mid["tickers"], raw_sm["tickers"]], ignore_index=True).drop_duplicates(["ticker", "cik"])
|
|
exm = ins._explode(tr, tm2)
|
|
colm = pd.Series(np.arange(len(pm.close.columns)), index=pm.close.columns)
|
|
exm = exm[exm["ticker"].isin(colm.index)].copy()
|
|
exm["col"] = colm.reindex(exm["ticker"]).values
|
|
group = ins.share_class_group(tm2)
|
|
B, Sl = exm["code"] == "P", exm["code"] == "S"
|
|
ev = {
|
|
"all purchases": ins.trade_events(pm, exm[B], group),
|
|
"opportunistic purchases": ins.trade_events(pm, exm[B & (exm["cmp"] == "opportunistic")], group),
|
|
"routine purchases": ins.trade_events(pm, exm[B & (exm["cmp"] == "routine")], group),
|
|
"all sales": ins.trade_events(pm, exm[Sl], group),
|
|
}
|
|
es = event_study(pm, ev)
|
|
done["event_study_midsmall"] = es
|
|
print("\n=== pooled S&P 400+600 event study: excess vs EW members, % (t clustered by month) [n]")
|
|
for kind, dd in es.items():
|
|
for h in HORIZONS:
|
|
cells = " ".join(f"{per}: {dd[per][f'+{h}']['mean']*100:+5.2f} ({dd[per][f'+{h}']['t']:+4.1f})"
|
|
f"[{dd[per][f'+{h}']['n']:5d}]" for per in EV_PERIODS)
|
|
print(f"{(kind if h == 5 else ''):26s} +{h:<3d} {cells}", flush=True)
|
|
|
|
with open(res_path, "w") as fh:
|
|
json.dump(done, fh, indent=2, default=float)
|
|
pd.DataFrame({k: v for k, v in rets.items()}).to_pickle(os.path.join(out_dir, "returns.pkl"))
|
|
from research.lab.trials import append_trials
|
|
append_trials(os.path.join("data", "research", "results", "trials_us.jsonl"),
|
|
[{"name": f"insider_mid[{uni}]:{k}", **{x: y for x, y in v.items() if x != "years"}}
|
|
for k, v in done.items() if k in rets],
|
|
{"study": "insider_midsmall", "role": "preregistered; holdout partly seen; not independent OOS"})
|
|
return done
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--universe", nargs="+", default=["mid", "small"], choices=["mid", "small"])
|
|
args = ap.parse_args()
|
|
for uni in args.universe:
|
|
run_universe(uni)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|