diff --git a/research/lab/activist_strategy.py b/research/lab/activist_strategy.py new file mode 100644 index 0000000..bb1d039 --- /dev/null +++ b/research/lab/activist_strategy.py @@ -0,0 +1,139 @@ +"""Activist event sleeve: hold fresh SC 13D targets, lab-engine evaluation. + +Event study first (research/lab/activists.py): +1.2% CAR1 excess in S&P +400/600 (t 4.2), +2.2% CAR21 in the S&P 500 (t 2.2); low-SVR events beat +high-SVR events (+3.6% vs +0.8% CAR21). + +Strategy: on each session, equal-weight all panel members whose SC 13D was +filed in the last H sessions (most recent K at most; daily rebalance to the +target basket, 20bp costs are built into the mid/small panels). PIT: the +weights row for day D includes filings with `filed <= D` and is executed at +the next open by the engine, i.e. after the filing is public. + + PYTHONPATH=. uv run python -m research.lab.activist_strategy [mid|small|midsmall|us] +""" +from __future__ import annotations + +import json +import os +import pickle +import sys + +import numpy as np +import pandas as pd + +from research.lab.activists import cik_ticker_map +from research.lab.engine import simulate +from research.lab.factor_scan import stats + +TARGETS = "data/research/edgar_13d_targets.csv" +SVR_PATH = "data/research/short_volume.pkl" + + +def load_events() -> pd.DataFrame: + tg = pd.read_csv(TARGETS) + tg = tg[tg["target_cik"].fillna(0) > 0].copy() + cik2t = cik_ticker_map() + tg["ticker"] = tg["target_cik"].map(lambda c: cik2t.get(int(c))) + tg = tg[tg["ticker"].notna()] + # fallback by exact name + if os.path.exists("data/research/company_tickers.json"): + raw = json.load(open("data/research/company_tickers.json")) + by_name = {str(v["title"]).strip().upper(): str(v["ticker"]).upper().replace(".", "-") + for v in raw.values()} + miss = tg["ticker"].isna() + tg.loc[miss, "ticker"] = tg.loc[miss, "target_name"].str.strip().str.upper().map(by_name) + tg = tg.sort_values("filed").drop_duplicates(subset=["ticker", "filed"], keep="first") + return tg[["ticker", "filed"]].dropna() + + +def svr21() -> pd.DataFrame | None: + if not os.path.exists(SVR_PATH): + return None + with open(SVR_PATH, "rb") as fh: + d = pickle.load(fh) + return (d["short"] / d["total"].replace(0.0, np.nan)).rolling(21, min_periods=10).mean() + + +def build_weights(p, events: pd.DataFrame, hold: int, k_max: int, + svr_filter: str | None = None, svr_lag: int = 2) -> pd.DataFrame: + dates = p.dates + cols = list(p.close.columns) + pos = {c: i for i, c in enumerate(cols)} + W = np.zeros((len(dates), len(cols))) + svr = svr21() + if svr is not None: + svr = svr.reindex(columns=cols).shift(svr_lag) + med = svr.where(p.member).median(axis=1) + + acts: list[tuple[int, int]] = [] # (start_idx, end_idx, col_idx) + for _, ev in events.iterrows(): + i = pos.get(ev["ticker"]) + if i is None: + continue + t0 = dates.searchsorted(pd.Timestamp(ev["filed"]), side="left") + if t0 >= len(dates) - 2: + continue + start, end = t0 + 1, min(t0 + 1 + hold, len(dates) - 1) + if svr_filter and svr is not None and dates[t0] in svr.index: + v = svr.at[dates[t0], ev["ticker"]] if ev["ticker"] in svr.columns else np.nan + m = med.get(dates[t0], np.nan) + if np.isfinite(v) and np.isfinite(m): + if svr_filter == "low" and v >= m: + continue + if svr_filter == "high" and v < m: + continue + acts.append((start, end, i)) + if not acts: + return pd.DataFrame(0.0, index=dates, columns=cols) + starts = np.array([a[0] for a in acts]) + ends = np.array([a[1] for a in acts]) + idx = np.array([a[2] for a in acts]) + for d in range(len(dates)): + active = np.where((starts <= d) & (d <= ends))[0] + if len(active) == 0: + continue + if k_max and len(active) > k_max: + active = active[np.argsort(-starts[active])][:k_max] # most recent filings + W[d, idx[active]] = 1.0 / len(active) + return pd.DataFrame(W, index=dates, columns=cols) + + +def run(mkt: str, events: pd.DataFrame, hold: int, k_max: int, + svr_filter: str | None = None) -> dict: + if mkt == "us": + with open("data/research/us.panel.pkl", "rb") as fh: + p = pickle.load(fh) + else: + from research.lab.panel_mid import load_us_mid + p = load_us_mid(mkt) + W = build_weights(p, events, hold, k_max, svr_filter) + r = simulate(p, target_weights=W, exec_at="open", min_trade=0.002) + blocks = {} + for a, b in (("2016-09-27", "2021-09-26"), ("2021-09-27", "2026-09-24")): + st = stats(r.equity.loc[a:b], r.turnover.loc[a:b]) + blocks[f"{a[:4]}-{b[:4]}"] = st + return {"mkt": mkt, "hold": hold, "k": k_max, "svr": svr_filter or "none", + "blocks": blocks} + + +def main() -> None: + mkts = sys.argv[1:] or ["midsmall"] + events = load_events() + print(f"events loaded: {len(events)}") + results = [] + for mkt in mkts: + for hold, k_max, svr_filter in ((21, 20, None), (21, 20, "low"), (63, 20, None), + (63, 20, "low"), (21, 10, None), (21, 10, "low")): + res = run(mkt, events, hold, k_max, svr_filter) + results.append(res) + line = f"{mkt:8s} H{hold:<3d} K{k_max:<3d} svr={res['svr']:4s}" + for blk, st in res["blocks"].items(): + line += f" | {blk} cagr {st['cagr']*100:5.1f}% sh {st['sharpe']:5.2f} mdd {st['mdd']*100:5.1f}%" + print(line) + json.dump(results, open("data/research/results/activist_sleeve.json", "w"), indent=1) + print("saved data/research/results/activist_sleeve.json") + + +if __name__ == "__main__": + main() diff --git a/research/lab/activists.py b/research/lab/activists.py new file mode 100644 index 0000000..ad7fc3d --- /dev/null +++ b/research/lab/activists.py @@ -0,0 +1,175 @@ +"""SC 13D activist-event study on the PIT US panels. + +Event: a new SC 13D filing (originals only; /A amendments excluded; 13G +passive stakes tracked separately) for a company whose CIK maps to a ticker +in the panel universe. EDGAR disseminates same-day, so the conservative +trade date is the next session; CARs run from close(t0) where t0 = first +session at/after the filing date. + +Reports market-adjusted forward returns (event minus same-panel member mean, +same window) over 1/5/21/63 sessions, split 13D vs 13G, with yearly means, +plus conditioning on 63d momentum quintile and SVR (short-volume ratio) +tercile to test the squeeze reading. + + PYTHONPATH=. uv run python -m research.lab.activists [us|midsmall|all] +""" +from __future__ import annotations + +import json +import os +import pickle +import sys + +import numpy as np +import pandas as pd +import requests + +from research.lab.fetch_us import _proxy_env + +EVENTS_CSV = "data/research/edgar_13d.csv" +TICKERS_CACHE = "data/research/company_tickers.json" +UA = {"User-Agent": "quant-research/0.1 gahow@example.com", "Accept-Encoding": "gzip"} +HORIZONS = (1, 5, 21, 63) + + +def cik_ticker_map() -> dict[int, str]: + if not os.path.exists(TICKERS_CACHE): + r = requests.get("https://www.sec.gov/files/company_tickers.json", headers=UA, timeout=60) + r.raise_for_status() + json.dump(r.json(), open(TICKERS_CACHE, "w")) + raw = json.load(open(TICKERS_CACHE)) + out = {} + for v in raw.values(): + t = str(v["ticker"]).upper().replace(".", "-") + out.setdefault(int(v["cik_str"]), t) + return out + + +def load_svr() -> pd.DataFrame | None: + path = "data/research/short_volume.pkl" + if not os.path.exists(path): + return None + with open(path, "rb") as fh: + d = pickle.load(fh) + return d["short"] / d["total"].replace(0.0, np.nan) + + +def run(name: str, close: pd.DataFrame, member: pd.DataFrame, + events: pd.DataFrame, svr: pd.DataFrame | None) -> dict: + close = close.ffill() + member = member.reindex_like(close).fillna(False) + idx = close.index + rets = close.pct_change(fill_method=None) + # cross-sectional member mean daily return (market proxy) + mkt = rets.where(member).mean(axis=1) + cum = (1 + rets.fillna(0.0)).cumprod() + cum_mkt = (1 + mkt.fillna(0.0)).cumprod() + + mom = close.pct_change(63) + svr21 = svr.rolling(21, min_periods=10).mean() if svr is not None else None + + rows = [] + for _, ev in events.iterrows(): + t = ev["ticker"] + if t not in close.columns: + continue + pos = idx.searchsorted(pd.Timestamp(ev["filed"]), side="left") + if pos >= len(idx) - max(HORIZONS) - 1: + continue + t0 = idx[pos] + if not bool(member.at[t0, t]): + continue + svr_val = np.nan + if svr21 is not None and t in svr21.columns and t0 in svr21.index: + v = svr21.at[t0, t] + if np.isfinite(v): + svr_val = float(v) + mom_val = mom.at[t0, t] if t in mom.columns else np.nan + entry = {"form": ev["form"], "ticker": t, "date": t0, "cik": ev["cik"], + "mom": float(mom_val) if np.isfinite(mom_val) else np.nan, + "svr": svr_val} + for h in HORIZONS: + r = float(cum.iloc[pos + h][t] / cum.iloc[pos][t] - 1.0) + rm = float(cum_mkt.iloc[pos + h] / cum_mkt.iloc[pos] - 1.0) + entry[f"car{h}"] = r - rm + rows.append(entry) + df = pd.DataFrame(rows) + print(f"\n=== {name}: {len(df)} events mapped to panel " + f"({df['date'].min().date() if len(df) else '-'}..{df['date'].max().date() if len(df) else '-'}) ===") + out = {"universe": name, "n_events": len(df)} + if len(df) == 0: + return out + for form in ("SC 13D", "SC 13G"): + sub = df[df["form"] == form] + if len(sub) < 30: + continue + line = [f"{form:7s} n={len(sub):5d}"] + for h in HORIZONS: + x = sub[f"car{h}"].dropna() + t = x.mean() / x.std() * np.sqrt(len(x)) if len(x) > 1 else np.nan + line.append(f"CAR{h}: {x.mean()*100:5.2f}% (t {t:4.1f}, hit {np.mean(x>0)*100:4.0f}%)") + print(" " + " ".join(line)) + out[form] = {f"car{h}": {"mean": float(sub[f'car{h}'].mean()), + "t": float(sub[f'car{h}'].mean() / sub[f'car{h}'].std() * np.sqrt(len(sub))), + "hit": float((sub[f'car{h}'] > 0).mean())} for h in HORIZONS} + yr = sub.groupby(sub["date"].dt.year)["car21"].mean() + print(" yearly CAR21: " + " ".join(f"{y}:{v*100:+.0f}%" for y, v in yr.items())) + # conditioning on momentum / svr (13D only) + d13 = df[df["form"] == "SC 13D"].copy() + if len(d13) >= 100: + d13["mom_q"] = d13["mom"].rank(pct=True) + hi, lo = d13[d13["mom_q"] > 0.66], d13[d13["mom_q"] < 0.34] + if len(hi) > 20 and len(lo) > 20: + print(f" 13D x momentum: hi-mom CAR21 {hi['car21'].mean()*100:5.2f}% (n={len(hi)}) " + f"| lo-mom CAR21 {lo['car21'].mean()*100:5.2f}% (n={len(lo)})") + out["13D_mom_hi_car21"] = float(hi["car21"].mean()) + out["13D_mom_lo_car21"] = float(lo["car21"].mean()) + if d13["svr"].notna().sum() > 100: + d13["svr_t"] = d13["svr"].rank(pct=True) + hi, lo = d13[d13["svr_t"] > 0.66], d13[d13["svr_t"] < 0.34] + if len(hi) > 20 and len(lo) > 20: + print(f" 13D x SVR: hi-fails-short CAR21 {hi['car21'].mean()*100:5.2f}% (n={len(hi)}) " + f"| lo CAR21 {lo['car21'].mean()*100:5.2f}% (n={len(lo)})") + out["13D_svr_hi_car21"] = float(hi["car21"].mean()) + out["13D_svr_lo_car21"] = float(lo["car21"].mean()) + return out + + +def main() -> None: + _proxy_env() + which = sys.argv[1] if len(sys.argv) > 1 else "all" + targets_csv = os.environ.get("TR9_TARGETS_CSV", "data/research/edgar_13d_targets.csv") + if not os.path.exists(targets_csv): + raise SystemExit("run research.lab.fetch_13d_targets first") + tg = pd.read_csv(targets_csv) + tg = tg[(tg["target_cik"].fillna(0) > 0)] + cik2t = cik_ticker_map() + tg["ticker"] = tg["target_cik"].map(lambda c: cik2t.get(int(c))) + # fallback: exact name match against the SEC ticker map + if os.path.exists(TICKERS_CACHE): + raw = json.load(open(TICKERS_CACHE)) + by_name = {str(v["title"]).strip().upper(): str(v["ticker"]).upper().replace(".", "-") + for v in raw.values()} + miss = tg["ticker"].isna() + tg.loc[miss, "ticker"] = tg.loc[miss, "target_name"].str.strip().str.upper().map(by_name) + ev = tg[tg["ticker"].notna()].copy() + ev["form"] = "SC 13D" + # one event per target per day (multiple filers on the same stake day) + ev = ev.sort_values("filed").drop_duplicates(subset=["ticker", "filed"], keep="first") + print(f"13D originals mapped to tickers: {len(ev)}") + svr = load_svr() + results = [] + if which in ("us", "all"): + with open("data/research/us.panel.pkl", "rb") as fh: + p = pickle.load(fh) + results.append(run("S&P 500", p.close, p.member, ev, svr)) + if which in ("midsmall", "all"): + with open("data/research/us_midsmall.panel.pkl", "rb") as fh: + p = pickle.load(fh) + results.append(run("S&P 400+600", p.close, p.member, ev, svr)) + json.dump(results, open("data/research/results/activists.json", "w"), indent=1, default=str) + print("saved data/research/results/activists.json") + + +if __name__ == "__main__": + main() diff --git a/research/lab/fails.py b/research/lab/fails.py new file mode 100644 index 0000000..717ff6a --- /dev/null +++ b/research/lab/fails.py @@ -0,0 +1,129 @@ +"""Fails-to-deliver factor scan on PIT US panels. + +FTD value / 20-day dollar volume = fails intensity: the share of a stock's +daily liquidity that is stuck unsettled. High intensity marks short-sale +constraint / squeeze fuel and, on the classic evidence, predicts negative +returns in small caps. + +PIT: SEC publishes the CNMS fails files roughly 3 weeks after settlement; +the scan shifts the FTD panel forward by 21 calendar days before +reindexing to trading days (ffill, staleness-capped). + + PYTHONPATH=. uv run python -m research.lab.fails [us|midsmall|all] +""" +from __future__ import annotations + +import pickle +import sys + +import numpy as np +import pandas as pd + +FAILS_PATH = "data/research/fails.pkl" +PUBLISH_LAG_DAYS = 21 + + +def load_ftd(close: pd.DataFrame, volume: pd.DataFrame | None) -> dict[str, pd.DataFrame]: + with open(FAILS_PATH, "rb") as fh: + d = pickle.load(fh) + shares, value = d["shares"], d["value"] + cols = [c for c in close.columns if c in shares.columns] + f = {} + v = value[cols] + f["ftd_value"] = v + f["ftd_shares"] = shares[cols] + if volume is not None and len(volume): + vol = volume.reindex(columns=cols).ffill() + px = close.reindex(columns=cols).ffill() + adv = (px * vol).rolling(20, min_periods=10).mean() + f["ftd_int"] = v / adv.replace(0.0, np.nan) + sh = shares[cols] + f["ftd_sh"] = sh / vol.rolling(20, min_periods=10).mean().replace(0.0, np.nan) + return f + + +def scan(name: str, close: pd.DataFrame, member: pd.DataFrame, + volume: pd.DataFrame | None) -> None: + raw = load_ftd(close, volume) + member = member.reindex_like(close).fillna(False) + print(f"\n=== {name}: {close.shape[1]} symbols, " + f"{close.index.min().date()}..{close.index.max().date()} ===") + fwd1 = close.shift(-1) / close - 1.0 + fwd21 = close.shift(-21) / close - 1.0 + months = fwd1.index.to_period("M") + rows = [] + lagged: dict[str, pd.DataFrame] = {} + for fname, fraw in raw.items(): + f = fraw.copy() + f.index = f.index + pd.Timedelta(days=PUBLISH_LAG_DAYS) + f = (f.reindex(index=close.index.union(f.index), columns=close.columns) + .ffill(limit=45).reindex(close.index)) + f = f.where(member) + lagged[fname] = f + ics1, ics21 = [], [] + for per in sorted(set(months)): + idx = months == per + xdf = f.loc[idx] + if xdf.notna().sum(axis=1).max() < 50: + continue + for ydf, out in ((fwd1.loc[idx], ics1), (fwd21.loc[idx], ics21)): + mask = xdf.notna().to_numpy() & np.isfinite(ydf.to_numpy()) + if mask.sum() < 100: + continue + xr = xdf.rank(axis=1).to_numpy()[mask] + yr = ydf.where(mask).rank(axis=1).to_numpy()[mask] + xm = xr - xr.mean() + ym = yr - yr.mean() + denom = np.sqrt((xm ** 2).sum() * (ym ** 2).sum()) + if denom > 0: + out.append(float((xm * ym).sum() / denom)) + ic1, ic21 = np.array(ics1), np.array(ics21) + rows.append({"factor": fname, + "IC1": ic1.mean(), "t1": ic1.mean() / ic1.std() * np.sqrt(len(ic1)) if len(ic1) > 1 else np.nan, + "IC21": ic21.mean(), "t21": ic21.mean() / ic21.std() * np.sqrt(len(ic21)) if len(ic21) > 1 else np.nan, + "months": len(ic21)}) + print(pd.DataFrame(rows).round(4).to_string(index=False)) + + for fname in [k for k in ("ftd_int", "ftd_sh") if k in lagged]: + f = lagged[fname] + pairs = [] + dates = f.index + for i in range(0, len(dates) - 21, 10): + t0 = dates[i] + x = f.loc[t0].dropna() + if len(x) < 50: + continue + q = pd.qcut(x.rank(method="first"), 5, labels=False) + lo, hi = q[q == 0].index, q[q == 4].index + r = close.iloc[i + 21] / close.iloc[i] - 1.0 + pairs.append((t0, float(r[hi].mean() - r[lo].mean()))) + if not pairs: + continue + s = np.array([p[1] for p in pairs]) + ys = pd.Series(s, index=pd.DatetimeIndex([p[0] for p in pairs])) + yearly = (1 + ys).groupby(ys.index.year).prod() - 1 + print(f"{fname}: Q5-Q1 21d spread mean {s.mean()*100:.2f}% " + f"t={s.mean()/s.std()*np.sqrt(len(s)):.2f} n={len(s)} yearly: " + + " ".join(f"{y}:{v*100:+.0f}%" for y, v in yearly.items())) + + +def main() -> None: + which = sys.argv[1] if len(sys.argv) > 1 else "all" + if which in ("us", "all"): + with open("data/research/us.panel.pkl", "rb") as fh: + p = pickle.load(fh) + vol = None + try: + vol = pd.read_csv("data/us_volume.csv", index_col=0, parse_dates=True) + except Exception: # noqa: BLE001 + pass + scan("S&P 500 ever-members", p.close, p.member, vol) + if which in ("midsmall", "all"): + with open("data/research/us_midsmall.panel.pkl", "rb") as fh: + p = pickle.load(fh) + vol = getattr(p, "volume", None) + scan("S&P 400+600 (midsmall)", p.close, p.member, vol) + + +if __name__ == "__main__": + main() diff --git a/research/lab/fetch_13d.py b/research/lab/fetch_13d.py new file mode 100644 index 0000000..3efe563 --- /dev/null +++ b/research/lab/fetch_13d.py @@ -0,0 +1,126 @@ +"""Fetch SC 13D / 13G filings (activist and 5% passive stakes) from EDGAR. + +Quarterly full-index files list every filing: + + https://www.sec.gov/Archives/edgar/full-index/YYYY/QTRn/form.idx + +Each is ~40-60 MB fixed-width text. Rows for SC 13D, SC 13D/A, SC 13G and +SC 13G/A are kept (filed date, CIK, company, archive path). Amendments are +flagged via the /A suffix so an event study can use originals only. + +Writes data/research/edgar_13d.csv (append-safe per quarter; a quarter +already present in the CSV is refetched only with --refresh). + + PYTHONPATH=. uv run python -m research.lab.fetch_13d [--quarters N] +""" +from __future__ import annotations + +import io +import os +import re +import sys +import time + +import pandas as pd +import requests + +from research.lab.fetch_us import _proxy_env + +OUT = os.path.join("data", "research", "edgar_13d.csv") +URL = "https://www.sec.gov/Archives/edgar/full-index/{y}/QTR{q}/form.idx" +UA = {"User-Agent": "quant-research/0.1 gahow@example.com", "Accept-Encoding": "gzip"} +ROW = re.compile( + r"^(?P