From dc14a541b0cc4d09e8f9288a19c2ac9cf7cb0d69 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 8 Oct 2026 09:54:56 -0700 Subject: [PATCH] research: separate S&P 400 sleeve family vs IJH (all REJECT) After S&P 400 names failed inside the large-cap momentum top 10, test them as their own sleeve: top 30, monthly, buffer 1, 20 bp, judged by alpha vs IJH and by whether 1/3 of the book in the sleeve beats 1/3 in IJH. Four hypotheses and a control, definitions and rule written before the run. Holdout 2017-01..2026-09: mid-cap momentum -1.0%/yr vs IJH (corr 0.83 with the large-cap book), lottery-excluded momentum -1.7%, SUE earnings momentum -1.7%, quality-screened momentum +3.5% (t 1.28), quality composite +3.4% (t 1.23; book 0.91 vs 0.85 with IJH). Quality is positive in both halves but underpowered; nothing is adopted. fetch_us_fundamentals gains --universe mid: EDGAR facts for S&P 400 ever-members, CIKs accepted only when an SEC name matches the iShares holding name. fundamentals.load_mid builds the same factors on that panel. Co-Authored-By: Claude Opus 5.5 --- research/lab/fetch_us_fundamentals.py | 79 ++++++++- research/lab/fundamentals.py | 21 ++- research/lab/midcap_sleeve.py | 220 ++++++++++++++++++++++++++ 3 files changed, 309 insertions(+), 11 deletions(-) create mode 100644 research/lab/midcap_sleeve.py diff --git a/research/lab/fetch_us_fundamentals.py b/research/lab/fetch_us_fundamentals.py index c875a46..2b20b2d 100644 --- a/research/lab/fetch_us_fundamentals.py +++ b/research/lab/fetch_us_fundamentals.py @@ -4,8 +4,9 @@ every S&P 500 ever-member in the research price panel, plus Yahoo split history (needed to turn split-adjusted prices back into market caps). uv run python -m research.lab.fetch_us_fundamentals + uv run python -m research.lab.fetch_us_fundamentals --universe mid # S&P 400 (IJH) ever-members -Writes data/research/us_fund_raw.pkl: +Writes data/research/us_fund_raw.pkl (us_mid_fund_raw.pkl for --universe mid): {"facts": DataFrame[ticker, tag, start, end, val, filed, form], "splits": DataFrame (dates x tickers, split ratio on split days)} @@ -26,6 +27,7 @@ import yfinance as yf from research.lab.fetch_us import _proxy_env OUT = os.path.join("data", "research", "us_fund_raw.pkl") +MID_OUT = os.path.join("data", "research", "us_mid_fund_raw.pkl") UA = {"User-Agent": "quant-research/0.1 gahow@example.com", "Accept-Encoding": "gzip"} FORMS = {"10-K", "10-Q", "10-K/A", "10-Q/A", "10-KT", "20-F", "40-F"} @@ -137,10 +139,79 @@ def fetch_splits(tickers: list[str], chunk: int = 60) -> tuple[pd.DataFrame, pd. return cat(splits).fillna(0.0), cat(closes) +_SUFFIXES = {"INC", "CORP", "CORPORATION", "CO", "COMPANY", "LTD", "PLC", "HOLDINGS", "HOLDING", + "GROUP", "THE", "CLASS", "CL", "A", "B", "NEW", "LLC", "LP", "SA", "NV", "TRUST"} + + +def _name_tokens(name: str) -> set[str]: + words = "".join(ch if ch.isalnum() else " " for ch in str(name).upper()).split() + return {w for w in words if w not in _SUFFIXES and len(w) > 1} + + +def mid_ciks(tickers: list[str]) -> tuple[dict[str, int], pd.DataFrame]: + """CIKs for S&P 400 tickers. Candidates come from sec_sic.csv and + mid_sec_sic.csv (earlier runs), then today's company_tickers.json; both + map a ticker to whoever holds it NOW, so a candidate counts only if one of + its SEC names (current or former) shares a word with an iShares holding + name of that ticker. Returns the accepted map and a per-ticker audit.""" + import json + from research.lab.fetch_us_mid import load_holdings + base = os.path.join("data", "research") + cand: dict[str, int] = {} + for f in ("sec_sic.csv", "mid_sec_sic.csv"): + d = pd.read_csv(os.path.join(base, f)).dropna(subset=["cik"]) + cand.update({t: int(c) for t, c in zip(d["ticker"], d["cik"]) if t not in cand}) + live = cik_map() + for t in tickers: + cand.setdefault(t, live.get(t, 0)) + names: dict[str, list[str]] = {} + for f in ("sec_names.json", "mid_sec_names.json"): + with open(os.path.join(base, f)) as fh: + names.update(json.load(fh)) + for t in tickers: + c = cand.get(t, 0) + if c and str(c) not in names: + r = requests.get(f"https://data.sec.gov/submissions/CIK{c:010d}.json", headers=UA, timeout=60) + time.sleep(0.15) + if r.ok: + js = r.json() + names[str(c)] = [js.get("name", "")] + [x.get("name", "") for x in js.get("formerNames", [])] + hold = load_holdings() + hnames = hold[hold["fund"] == "IJH"].groupby("ticker")["name"].apply(lambda s: set(s.dropna())) + rows, ok = [], {} + for t in tickers: + c = cand.get(t, 0) + sec = names.get(str(c), []) if c else [] + hw = set().union(*[_name_tokens(n) for n in hnames.get(t, set())]) if t in hnames else set() + sw = set().union(*[_name_tokens(n) for n in sec]) if sec else set() + # token overlap, or the same leading letters once spacing and + # punctuation are gone (MACYS / Macy's, SIRIUSXM / Sirius XM, VF / V F) + squash = lambda n: "".join(w for w in "".join(ch if ch.isalnum() else " " for ch in str(n).upper()).split() + if w not in _SUFFIXES) + hs = {squash(n)[:5] for n in hnames.get(t, set())} - {""} + ss = {squash(n)[:5] for n in sec} - {""} + match = bool(hw & sw) or bool(hs & ss) + if c and match: + ok[t] = c + rows.append({"ticker": t, "cik": c or None, "ishares_names": " | ".join(sorted(hnames.get(t, set()))), + "sec_names": " | ".join(sec), "accepted": bool(c and match)}) + return ok, pd.DataFrame(rows) + + def main() -> None: import sys - with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh: - tickers = [t for t in pickle.load(fh)["close"].columns] + global OUT + if "--universe" in sys.argv and sys.argv[sys.argv.index("--universe") + 1] == "mid": + from research.lab.panel_mid import load_us_mid + OUT = MID_OUT + tickers = list(load_us_mid("mid").close.columns) + ciks, audit = mid_ciks(tickers) + audit.to_csv(os.path.join("data", "research", "us_mid_fund_cik_audit.csv"), index=False) + print(f"{len(ciks)}/{len(tickers)} S&P 400 tickers have a name-checked CIK", flush=True) + else: + with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh: + tickers = [t for t in pickle.load(fh)["close"].columns] + ciks = None if "--yahoo-only" in sys.argv: # refresh splits/closes of an existing file with open(OUT, "rb") as fh: saved = pickle.load(fh) @@ -150,7 +221,7 @@ def main() -> None: pickle.dump(saved, fh) print(f"splits: {int((saved['splits'] > 0).sum().sum())} events; saved {OUT}") return - ciks = cik_map() + ciks = ciks if ciks is not None else cik_map() todo = [(t, ciks[t]) for t in tickers if t in ciks] print(f"{len(todo)}/{len(tickers)} tickers map to a CIK", flush=True) diff --git a/research/lab/fundamentals.py b/research/lab/fundamentals.py index b842260..c03b6ea 100644 --- a/research/lab/fundamentals.py +++ b/research/lab/fundamentals.py @@ -23,6 +23,8 @@ import pandas as pd RAW = os.path.join("data", "research", "us_fund_raw.pkl") CACHE = os.path.join("data", "research", "us_fund_factors.pkl") +MID_RAW = os.path.join("data", "research", "us_mid_fund_raw.pkl") # fetch_us_fundamentals --universe mid +MID_CACHE = os.path.join("data", "research", "us_mid_fund_factors.pkl") DURATION = { "revenue": ["Revenues", "RevenueFromContractWithCustomerExcludingAssessedTax", @@ -143,8 +145,8 @@ def _series_by_ticker(df: pd.DataFrame) -> dict[str, pd.DataFrame]: # build # --------------------------------------------------------------------------- -def build(p) -> dict[str, pd.DataFrame]: - with open(RAW, "rb") as fh: +def build(p, raw_path: str = RAW) -> dict[str, pd.DataFrame]: + with open(raw_path, "rb") as fh: raw = pickle.load(fh) facts = raw["facts"] dates, tickers = p.dates, list(p.close.columns) @@ -282,11 +284,16 @@ def build(p) -> dict[str, pd.DataFrame]: return {k: v.astype("float32") for k, v in F.items()} -def load(p, rebuild: bool = False) -> dict[str, pd.DataFrame]: - if os.path.exists(CACHE) and not rebuild: - with open(CACHE, "rb") as fh: +def load(p, rebuild: bool = False, raw_path: str = RAW, cache: str = CACHE) -> dict[str, pd.DataFrame]: + if os.path.exists(cache) and not rebuild: + with open(cache, "rb") as fh: return pickle.load(fh) - F = build(p) - with open(CACHE, "wb") as fh: + F = build(p, raw_path) + with open(cache, "wb") as fh: pickle.dump(F, fh, protocol=4) return F + + +def load_mid(p, rebuild: bool = False) -> dict[str, pd.DataFrame]: + """The same factors for the S&P 400 panel (research/lab/panel_mid.py).""" + return load(p, rebuild, MID_RAW, MID_CACHE) diff --git a/research/lab/midcap_sleeve.py b/research/lab/midcap_sleeve.py new file mode 100644 index 0000000..23a2f9e --- /dev/null +++ b/research/lab/midcap_sleeve.py @@ -0,0 +1,220 @@ +""" +A separate S&P 400 sleeve: does any of four preregistered signals earn +alpha over IJH and improve the book more than holding IJH would? + + PYTHONPATH=. uv run python -m research.lab.midcap_sleeve --only h4 + PYTHONPATH=. uv run python -m research.lab.midcap_sleeve --only h1 h2 h3 # needs us_mid_fund_raw.pkl + +Background (research/lab/universe_midcap.py): ranking S&P 400 names inside +the large-cap momentum top 10 lost (Sharpe 0.86 vs 0.98, 2019-26); the S&P +400 winners were a few lottery tickets among many small losers. Here the +mid caps get their own book and their own benchmark. + +Every sleeve: PIT S&P 400 members (research/lab/panel_mid.py, Tiingo- +extended), top 30, equal weight, rebalance every 21 sessions with buffer 1 +(a holding stays while its rank is within 60), next-open fills, 20 bp a +side, averaged over the 21 rebalance phases. Definitions and the decision +rule live in PREREG and are written to disk before the first run; they +are textbook choices, not fitted (price coverage before 2017 is 46-78% and +XBRL starts in 2009-11, so there is no clean design period to fit on). + +The book used for the portfolio test is the lab replay of +overnight_recovery_top10 (S&P 500, 20 bp) saved by universe_midcap.py. + +Result (2026-10-08): all four REJECT. Holdout 2017-01..2026-09 (book alone +Sharpe 0.88; 2/3 book + 1/3 IJH 0.85): + + sleeve CAGR Sharpe alpha/IJH (t) halves corr book book+sleeve + c0 momentum (control) 7.1% 0.46 -1.0% (-0.39) -0.9 / -1.0 0.83 0.80 + h1 earnings momentum 7.8% 0.45 -1.7% (-0.74) +1.0 / -4.3 0.60 0.83 + h2 quality-screened momentum 14.0% 0.65 +3.5% (1.28) +5.8 / +1.6 0.70 0.86 + h3 quality composite 13.2% 0.67 +3.4% (1.23) +4.3 / +2.4 0.52 0.91 + h4 lottery-excluded momentum 6.2% 0.46 -1.7% (-1.07) -0.1 / -3.8 0.71 0.83 + EW members 11.3% 0.59 +1.0% (1.18) + IJH 10.1% 0.56 + +Price momentum among S&P 400 names loses to the index and moves with the +large-cap book (corr 0.83), so it diversifies nothing. Quality (h2, h3) is +the only family with positive alpha in both halves and the only one that +helps the book (h3: 0.91 vs 0.85, corr 0.52), but at ~3.5%/yr with ~10% +tracking error t is 1.2-1.3; t 2 would need about 25 years at this effect +size. Boundary: gp_a covers 55-70% of members (no gross profit for banks, +REITs), so h2's median screen also drops most financials. EDGAR CIKs are +name-checked against iShares holding names (723/845 tickers; members +covered 84-98% a year from 2017). +""" + +from __future__ import annotations + +import argparse +import json +import os +import warnings + +import numpy as np +import pandas as pd + +from research.lab.alpha import alpha_stats +from research.lab.factors import overnight_mom, rec_126 +from research.lab.panel_mid import load_us_mid +from research.lab.us_final import monthly_blend +from research.lab.us_sweep import _blend, run + +warnings.filterwarnings("ignore") +OUT = os.path.join("data", "research", "results", "midcap_sleeve") +BOOK = os.path.join("data", "research", "results", "universe_midcap", "returns.pkl") +HOLDOUT = ("2017-01-03", "2026-09-24") +HALVES = [("2017-01-03", "2021-12-31"), ("2022-01-03", "2026-09-24")] +REFERENCE = ("2010-01-04", "2016-12-30") +KW = dict(top_n=30, buffer=1.0, every=21) +PREREG = { + "family": "S&P 400 sleeve, 4 hypotheses + 1 control; all run once, results reported whatever they are", + "construction": "PIT S&P 400 members, top 30, equal weight, every 21 sessions, buffer 1, next-open fills, " + "20 bp a side, 21 phases averaged", + "signals": { + "c0_momentum": "control: mean pct rank of overnight_mom (252d sum of log open/prev close, skip 21) " + "and rec_126 (close / 126d low - 1) among members (the live overnight_recovery score)", + "h1_earnings_momentum": "mean pct rank of sue and sue_rev (research/lab/fundamentals.py: seasonal " + "change scaled by the std of the last 8, visible from the filing date)", + "h2_quality_screened_momentum": "c0 score, eligible only if roa (TTM net income / assets) > 0 and " + "gp_a >= the members' median that day", + "h3_quality_composite": "mean pct rank of gp_a, roa, accruals (-(NI-CFO)/assets), net_issuance " + "(-12m log change in shares)", + "h4_lottery_excluded_momentum": "c0 score, eligible only if MAX (largest daily return, last 21 " + "sessions) and idiosyncratic vol (63d, residual to IJH) are both " + "below the members' 80th percentile that day", + }, + "holdout": HOLDOUT, + "accept_if_all": [ + "A1: alpha vs IJH over the holdout with Newey-West t >= 2.0, or t >= 1.5 and alpha > 0 in both " + "2017-2021 and 2022-2026", + "A2: Sharpe(2/3 book + 1/3 sleeve) > Sharpe(2/3 book + 1/3 IJH), monthly reset, holdout", + "A3 (h2, h4 only): Sharpe(sleeve) > Sharpe(c0_momentum), holdout", + ], + "on_accept": "forward-only validation book; no production change", + "multiple_testing": "4 hypotheses: a lone t just above 2 is weak evidence; report all of them", + "reference_only": f"{REFERENCE}: price coverage 57-78%, survivor-biased", + "status": "holdout overlaps periods seen in earlier mid/small scans; not independent OOS", +} + + +def lottery_ok(p, members: pd.DataFrame) -> pd.DataFrame: + """True where MAX(21) and idiosyncratic vol(63) are both under the + members' 80th percentile. Idio variance = var(r) - beta^2 var(m).""" + r = p.close.pct_change(fill_method=None) + m = p.extra["etf"]["IJH"].pct_change(fill_method=None).reindex(p.dates) + mx = r.rolling(21, min_periods=15).max() + var_r = r.rolling(63, min_periods=40).var() + var_m = m.rolling(63, min_periods=40).var() + cov = r.rolling(63, min_periods=40).cov(m) + idio = np.sqrt((var_r - cov.pow(2).div(var_m, axis=0)).clip(lower=0)) + rank = lambda x: x.where(members).rank(axis=1, pct=True) + return (rank(mx) <= 0.8) & (rank(idio) <= 0.8) + + +def score(factors: list[pd.DataFrame], eligible: pd.DataFrame) -> pd.DataFrame: + return _blend(*[f.where(eligible).rank(axis=1, pct=True) for f in factors]).where(eligible) + + +def sleeves(p, which: list[str]) -> dict[str, pd.DataFrame]: + mem = p.member + mom = [overnight_mom(p), rec_126(p)] + out = {"c0_momentum": score(mom, mem)} + if "h4" in which: + out["h4_lottery_excluded_momentum"] = score(mom, mem & lottery_ok(p, mem)) + if {"h1", "h2", "h3"} & set(which): + from research.lab import fundamentals as fu + F = fu.load_mid(p) + if "h1" in which: + out["h1_earnings_momentum"] = score([F["sue"], F["sue_rev"]], mem) + if "h2" in which: + gp = F["gp_a"].where(mem) + ok = (F["roa"] > 0) & gp.ge(gp.median(axis=1), axis=0) + out["h2_quality_screened_momentum"] = score(mom, mem & ok) + if "h3" in which: + out["h3_quality_composite"] = score([F["gp_a"], F["roa"], F["accruals"], F["net_issuance"]], mem) + return out + + +def sharpe(r: pd.Series) -> float: + return float(r.mean() / r.std() * np.sqrt(252)) + + +def describe(r: pd.Series, ijh: pd.Series, book: pd.Series) -> dict: + h = r.loc[HOLDOUT[0]:HOLDOUT[1]] + e = (1 + h).cumprod() + a = alpha_stats(h, ijh) + halves = [alpha_stats(r.loc[x:y], ijh)["alpha"] for x, y in HALVES] + R = pd.DataFrame({"book": book, "s": r, "ijh": ijh}).loc[HOLDOUT[0]:HOLDOUT[1]].dropna() + with_s = monthly_blend(R, {"book": 2 / 3, "s": 1 / 3}) + with_i = monthly_blend(R, {"book": 2 / 3, "ijh": 1 / 3}) + ref = r.loc[REFERENCE[0]:REFERENCE[1]] + return {"cagr": float(e.iloc[-1] ** (252 / len(h)) - 1), "sharpe": sharpe(h), + "mdd": float((e / e.cummax() - 1).min()), + "alpha_ijh": float(a["alpha"]), "alpha_t": float(a["t"]), "beta_ijh": float(a["beta"]), + "alpha_halves": [float(x) for x in halves], + "corr_book": float(R["s"].corr(R["book"])), + "book_sharpe_with_sleeve": sharpe(with_s), "book_sharpe_with_ijh": sharpe(with_i), + "reference_sharpe": sharpe(ref), + "years": {int(k): float(v) for k, v in ((1 + h).groupby(h.index.year).prod() - 1).items()}} + + +def verdict(name: str, d: dict, c0: dict) -> dict: + a1 = d["alpha_t"] >= 2.0 or (d["alpha_t"] >= 1.5 and min(d["alpha_halves"]) > 0) + a2 = d["book_sharpe_with_sleeve"] > d["book_sharpe_with_ijh"] + checks = {"A1": a1, "A2": a2} + if name.startswith(("h2", "h4")): + checks["A3"] = d["sharpe"] > c0["sharpe"] + return {"checks": {k: bool(v) for k, v in checks.items()}, + "verdict": "ACCEPT" if all(checks.values()) else "REJECT"} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--only", nargs="+", default=["h1", "h2", "h3", "h4"]) + which = ap.parse_args().only + os.makedirs(OUT, exist_ok=True) + pre = os.path.join(OUT, "prereg.json") + if not os.path.exists(pre): + with open(pre, "w") as fh: + json.dump(PREREG, fh, indent=2) + res_path = os.path.join(OUT, "results.json") + done = json.load(open(res_path)) if os.path.exists(res_path) else {} + clash = [k for k in done if k != "c0_momentum" and k.split("_")[0] in which] + if clash: + raise SystemExit(f"already run: {clash}; this study does not overwrite results") + + p = load_us_mid("mid") + ijh = p.extra["etf"]["IJH"].pct_change(fill_method=None) + book = pd.read_pickle(BOOK)["base"] + S = sleeves(p, which) + span = [(REFERENCE[0], HOLDOUT[1])] + rets = {} + for name, sc in S.items(): + rets[name] = run(p, sc, span, **KW)["_rets"] + print(f"ran {name}", flush=True) + ew = p.close.pct_change(fill_method=None).where(p.member.shift(1, fill_value=False)).mean(axis=1) + rets["bench_ew_members"] = ew + rets["bench_ijh"] = ijh + for name, r in rets.items(): + d = describe(r.reindex(p.dates).fillna(0.0), ijh, book) + if name.startswith("h"): + d.update(verdict(name, d, done.get("c0_momentum") or describe(rets["c0_momentum"], ijh, book))) + done[name] = d + print(f"{name:32s} cagr {d['cagr']*100:5.1f}% sh {d['sharpe']:.2f} mdd {d['mdd']*100:6.1f}% " + f"alpha {d['alpha_ijh']*100:5.1f}% t {d['alpha_t']:.2f} halves " + f"{[round(x*100, 1) for x in d['alpha_halves']]} corr_book {d['corr_book']:.2f} " + f"book+sleeve {d['book_sharpe_with_sleeve']:.2f} vs book+IJH {d['book_sharpe_with_ijh']:.2f} " + f"ref {d['reference_sharpe']:.2f} {d.get('verdict', '')} {d.get('checks', '')}") + with open(res_path, "w") as fh: + json.dump(done, fh, indent=2) + pd.DataFrame(rets).to_pickle(os.path.join(OUT, f"returns_{'_'.join(which)}.pkl")) + from research.lab.trials import append_trials + append_trials(os.path.join("data", "research", "results", "trials_us.jsonl"), + [{"name": f"midcap_sleeve:{k}", **{x: y for x, y in v.items() if x != "years"}} + for k, v in done.items() if k in rets], + {"study": "midcap_sleeve", "role": "preregistered family; holdout partly seen"}) + + +if __name__ == "__main__": + main()