gahow/quant-trading
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: separate S&P 400 sleeve family vs IJH (all REJECT)

Browse Source 
After S&P 400 names failed inside the large-cap momentum top 10, test them
as their own sleeve: top 30, monthly, buffer 1, 20 bp, judged by alpha vs
IJH and by whether 1/3 of the book in the sleeve beats 1/3 in IJH. Four
hypotheses and a control, definitions and rule written before the run.

Holdout 2017-01..2026-09: mid-cap momentum -1.0%/yr vs IJH (corr 0.83 with
the large-cap book), lottery-excluded momentum -1.7%, SUE earnings momentum
-1.7%, quality-screened momentum +3.5% (t 1.28), quality composite +3.4%
(t 1.23; book 0.91 vs 0.85 with IJH). Quality is positive in both halves
but underpowered; nothing is adopted.

fetch_us_fundamentals gains --universe mid: EDGAR facts for S&P 400
ever-members, CIKs accepted only when an SEC name matches the iShares
holding name. fundamentals.load_mid builds the same factors on that panel.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-10-08 09:54:56 -07:00
parent 92af4e5ac0
commit dc14a541b0
3 changed files with 309 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
research/lab/fetch_us_fundamentals.py
View File

@@ -4,8 +4,9 @@ every S&P 500 ever-member in the research price panel, plus Yahoo split
history (needed to turn split-adjusted prices back into market caps).
uv run python -m research.lab.fetch_us_fundamentals
uv run python -m research.lab.fetch_us_fundamentals --universe mid # S&P 400 (IJH) ever-members
Writes data/research/us_fund_raw.pkl:
Writes data/research/us_fund_raw.pkl (us_mid_fund_raw.pkl for --universe mid):
{"facts": DataFrame[ticker, tag, start, end, val, filed, form],
"splits": DataFrame (dates x tickers, split ratio on split days)}
@@ -26,6 +27,7 @@ import yfinance as yf
from research.lab.fetch_us import _proxy_env
OUT = os.path.join("data", "research", "us_fund_raw.pkl")
MID_OUT = os.path.join("data", "research", "us_mid_fund_raw.pkl")
UA = {"User-Agent": "quant-research/0.1 gahow@example.com", "Accept-Encoding": "gzip"}
FORMS = {"10-K", "10-Q", "10-K/A", "10-Q/A", "10-KT", "20-F", "40-F"}
@@ -137,10 +139,79 @@ def fetch_splits(tickers: list[str], chunk: int = 60) -> tuple[pd.DataFrame, pd.
return cat(splits).fillna(0.0), cat(closes)
_SUFFIXES = {"INC", "CORP", "CORPORATION", "CO", "COMPANY", "LTD", "PLC", "HOLDINGS", "HOLDING",
"GROUP", "THE", "CLASS", "CL", "A", "B", "NEW", "LLC", "LP", "SA", "NV", "TRUST"}
def _name_tokens(name: str) -> set[str]:
words = "".join(ch if ch.isalnum() else " " for ch in str(name).upper()).split()
return {w for w in words if w not in _SUFFIXES and len(w) > 1}
def mid_ciks(tickers: list[str]) -> tuple[dict[str, int], pd.DataFrame]:
"""CIKs for S&P 400 tickers. Candidates come from sec_sic.csv and
mid_sec_sic.csv (earlier runs), then today's company_tickers.json; both
map a ticker to whoever holds it NOW, so a candidate counts only if one of
its SEC names (current or former) shares a word with an iShares holding
name of that ticker. Returns the accepted map and a per-ticker audit."""
import json
from research.lab.fetch_us_mid import load_holdings
base = os.path.join("data", "research")
cand: dict[str, int] = {}
for f in ("sec_sic.csv", "mid_sec_sic.csv"):
d = pd.read_csv(os.path.join(base, f)).dropna(subset=["cik"])
cand.update({t: int(c) for t, c in zip(d["ticker"], d["cik"]) if t not in cand})
live = cik_map()
for t in tickers:
cand.setdefault(t, live.get(t, 0))
names: dict[str, list[str]] = {}
for f in ("sec_names.json", "mid_sec_names.json"):
with open(os.path.join(base, f)) as fh:
names.update(json.load(fh))
for t in tickers:
c = cand.get(t, 0)
if c and str(c) not in names:
r = requests.get(f"https://data.sec.gov/submissions/CIK{c:010d}.json", headers=UA, timeout=60)
time.sleep(0.15)
if r.ok:
js = r.json()
names[str(c)] = [js.get("name", "")] + [x.get("name", "") for x in js.get("formerNames", [])]
hold = load_holdings()
hnames = hold[hold["fund"] == "IJH"].groupby("ticker")["name"].apply(lambda s: set(s.dropna()))
rows, ok = [], {}
for t in tickers:
c = cand.get(t, 0)
sec = names.get(str(c), []) if c else []
hw = set().union(*[_name_tokens(n) for n in hnames.get(t, set())]) if t in hnames else set()
sw = set().union(*[_name_tokens(n) for n in sec]) if sec else set()
# token overlap, or the same leading letters once spacing and
# punctuation are gone (MACYS / Macy's, SIRIUSXM / Sirius XM, VF / V F)
squash = lambda n: "".join(w for w in "".join(ch if ch.isalnum() else " " for ch in str(n).upper()).split()
if w not in _SUFFIXES)
hs = {squash(n)[:5] for n in hnames.get(t, set())} - {""}
ss = {squash(n)[:5] for n in sec} - {""}
match = bool(hw & sw) or bool(hs & ss)
if c and match:
ok[t] = c
rows.append({"ticker": t, "cik": c or None, "ishares_names": " | ".join(sorted(hnames.get(t, set()))),
"sec_names": " | ".join(sec), "accepted": bool(c and match)})
return ok, pd.DataFrame(rows)
def main() -> None:
import sys
with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh:
tickers = [t for t in pickle.load(fh)["close"].columns]
global OUT
if "--universe" in sys.argv and sys.argv[sys.argv.index("--universe") + 1] == "mid":
from research.lab.panel_mid import load_us_mid
OUT = MID_OUT
tickers = list(load_us_mid("mid").close.columns)
ciks, audit = mid_ciks(tickers)
audit.to_csv(os.path.join("data", "research", "us_mid_fund_cik_audit.csv"), index=False)
print(f"{len(ciks)}/{len(tickers)} S&P 400 tickers have a name-checked CIK", flush=True)
else:
with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh:
tickers = [t for t in pickle.load(fh)["close"].columns]
ciks = None
if "--yahoo-only" in sys.argv: # refresh splits/closes of an existing file
with open(OUT, "rb") as fh:
saved = pickle.load(fh)
@@ -150,7 +221,7 @@ def main() -> None:
pickle.dump(saved, fh)
print(f"splits: {int((saved['splits'] > 0).sum().sum())} events; saved {OUT}")
return
ciks = cik_map()
ciks = ciks if ciks is not None else cik_map()
todo = [(t, ciks[t]) for t in tickers if t in ciks]
print(f"{len(todo)}/{len(tickers)} tickers map to a CIK", flush=True)

21
research/lab/fundamentals.py
View File

@@ -23,6 +23,8 @@ import pandas as pd
RAW = os.path.join("data", "research", "us_fund_raw.pkl")
CACHE = os.path.join("data", "research", "us_fund_factors.pkl")
MID_RAW = os.path.join("data", "research", "us_mid_fund_raw.pkl") # fetch_us_fundamentals --universe mid
MID_CACHE = os.path.join("data", "research", "us_mid_fund_factors.pkl")
DURATION = {
"revenue": ["Revenues", "RevenueFromContractWithCustomerExcludingAssessedTax",
@@ -143,8 +145,8 @@ def _series_by_ticker(df: pd.DataFrame) -> dict[str, pd.DataFrame]:
# build
# ---------------------------------------------------------------------------
def build(p) -> dict[str, pd.DataFrame]:
with open(RAW, "rb") as fh:
def build(p, raw_path: str = RAW) -> dict[str, pd.DataFrame]:
with open(raw_path, "rb") as fh:
raw = pickle.load(fh)
facts = raw["facts"]
dates, tickers = p.dates, list(p.close.columns)
@@ -282,11 +284,16 @@ def build(p) -> dict[str, pd.DataFrame]:
return {k: v.astype("float32") for k, v in F.items()}
def load(p, rebuild: bool = False) -> dict[str, pd.DataFrame]:
if os.path.exists(CACHE) and not rebuild:
with open(CACHE, "rb") as fh:
def load(p, rebuild: bool = False, raw_path: str = RAW, cache: str = CACHE) -> dict[str, pd.DataFrame]:
if os.path.exists(cache) and not rebuild:
with open(cache, "rb") as fh:
return pickle.load(fh)
F = build(p)
with open(CACHE, "wb") as fh:
F = build(p, raw_path)
with open(cache, "wb") as fh:
pickle.dump(F, fh, protocol=4)
return F
def load_mid(p, rebuild: bool = False) -> dict[str, pd.DataFrame]:
"""The same factors for the S&P 400 panel (research/lab/panel_mid.py)."""
return load(p, rebuild, MID_RAW, MID_CACHE)

220
research/lab/midcap_sleeve.py Normal file
View File

@@ -0,0 +1,220 @@
"""
A separate S&P 400 sleeve: does any of four preregistered signals earn
alpha over IJH and improve the book more than holding IJH would?
PYTHONPATH=. uv run python -m research.lab.midcap_sleeve --only h4
PYTHONPATH=. uv run python -m research.lab.midcap_sleeve --only h1 h2 h3 # needs us_mid_fund_raw.pkl
Background (research/lab/universe_midcap.py): ranking S&P 400 names inside
the large-cap momentum top 10 lost (Sharpe 0.86 vs 0.98, 2019-26); the S&P
400 winners were a few lottery tickets among many small losers. Here the
mid caps get their own book and their own benchmark.
Every sleeve: PIT S&P 400 members (research/lab/panel_mid.py, Tiingo-
extended), top 30, equal weight, rebalance every 21 sessions with buffer 1
(a holding stays while its rank is within 60), next-open fills, 20 bp a
side, averaged over the 21 rebalance phases. Definitions and the decision
rule live in PREREG and are written to disk before the first run; they
are textbook choices, not fitted (price coverage before 2017 is 46-78% and
XBRL starts in 2009-11, so there is no clean design period to fit on).
The book used for the portfolio test is the lab replay of
overnight_recovery_top10 (S&P 500, 20 bp) saved by universe_midcap.py.
Result (2026-10-08): all four REJECT. Holdout 2017-01..2026-09 (book alone
Sharpe 0.88; 2/3 book + 1/3 IJH 0.85):
sleeve CAGR Sharpe alpha/IJH (t) halves corr book book+sleeve
c0 momentum (control) 7.1% 0.46 -1.0% (-0.39) -0.9 / -1.0 0.83 0.80
h1 earnings momentum 7.8% 0.45 -1.7% (-0.74) +1.0 / -4.3 0.60 0.83
h2 quality-screened momentum 14.0% 0.65 +3.5% (1.28) +5.8 / +1.6 0.70 0.86
h3 quality composite 13.2% 0.67 +3.4% (1.23) +4.3 / +2.4 0.52 0.91
h4 lottery-excluded momentum 6.2% 0.46 -1.7% (-1.07) -0.1 / -3.8 0.71 0.83
EW members 11.3% 0.59 +1.0% (1.18)
IJH 10.1% 0.56
Price momentum among S&P 400 names loses to the index and moves with the
large-cap book (corr 0.83), so it diversifies nothing. Quality (h2, h3) is
the only family with positive alpha in both halves and the only one that
helps the book (h3: 0.91 vs 0.85, corr 0.52), but at ~3.5%/yr with ~10%
tracking error t is 1.2-1.3; t 2 would need about 25 years at this effect
size. Boundary: gp_a covers 55-70% of members (no gross profit for banks,
REITs), so h2's median screen also drops most financials. EDGAR CIKs are
name-checked against iShares holding names (723/845 tickers; members
covered 84-98% a year from 2017).
"""
from __future__ import annotations
import argparse
import json
import os
import warnings
import numpy as np
import pandas as pd
from research.lab.alpha import alpha_stats
from research.lab.factors import overnight_mom, rec_126
from research.lab.panel_mid import load_us_mid
from research.lab.us_final import monthly_blend
from research.lab.us_sweep import _blend, run
warnings.filterwarnings("ignore")
OUT = os.path.join("data", "research", "results", "midcap_sleeve")
BOOK = os.path.join("data", "research", "results", "universe_midcap", "returns.pkl")
HOLDOUT = ("2017-01-03", "2026-09-24")
HALVES = [("2017-01-03", "2021-12-31"), ("2022-01-03", "2026-09-24")]
REFERENCE = ("2010-01-04", "2016-12-30")
KW = dict(top_n=30, buffer=1.0, every=21)
PREREG = {
"family": "S&P 400 sleeve, 4 hypotheses + 1 control; all run once, results reported whatever they are",
"construction": "PIT S&P 400 members, top 30, equal weight, every 21 sessions, buffer 1, next-open fills, "
"20 bp a side, 21 phases averaged",
"signals": {
"c0_momentum": "control: mean pct rank of overnight_mom (252d sum of log open/prev close, skip 21) "
"and rec_126 (close / 126d low - 1) among members (the live overnight_recovery score)",
"h1_earnings_momentum": "mean pct rank of sue and sue_rev (research/lab/fundamentals.py: seasonal "
"change scaled by the std of the last 8, visible from the filing date)",
"h2_quality_screened_momentum": "c0 score, eligible only if roa (TTM net income / assets) > 0 and "
"gp_a >= the members' median that day",
"h3_quality_composite": "mean pct rank of gp_a, roa, accruals (-(NI-CFO)/assets), net_issuance "
"(-12m log change in shares)",
"h4_lottery_excluded_momentum": "c0 score, eligible only if MAX (largest daily return, last 21 "
"sessions) and idiosyncratic vol (63d, residual to IJH) are both "
"below the members' 80th percentile that day",
},
"holdout": HOLDOUT,
"accept_if_all": [
"A1: alpha vs IJH over the holdout with Newey-West t >= 2.0, or t >= 1.5 and alpha > 0 in both "
"2017-2021 and 2022-2026",
"A2: Sharpe(2/3 book + 1/3 sleeve) > Sharpe(2/3 book + 1/3 IJH), monthly reset, holdout",
"A3 (h2, h4 only): Sharpe(sleeve) > Sharpe(c0_momentum), holdout",
],
"on_accept": "forward-only validation book; no production change",
"multiple_testing": "4 hypotheses: a lone t just above 2 is weak evidence; report all of them",
"reference_only": f"{REFERENCE}: price coverage 57-78%, survivor-biased",
"status": "holdout overlaps periods seen in earlier mid/small scans; not independent OOS",
}
def lottery_ok(p, members: pd.DataFrame) -> pd.DataFrame:
"""True where MAX(21) and idiosyncratic vol(63) are both under the
members' 80th percentile. Idio variance = var(r) - beta^2 var(m)."""
r = p.close.pct_change(fill_method=None)
m = p.extra["etf"]["IJH"].pct_change(fill_method=None).reindex(p.dates)
mx = r.rolling(21, min_periods=15).max()
var_r = r.rolling(63, min_periods=40).var()
var_m = m.rolling(63, min_periods=40).var()
cov = r.rolling(63, min_periods=40).cov(m)
idio = np.sqrt((var_r - cov.pow(2).div(var_m, axis=0)).clip(lower=0))
rank = lambda x: x.where(members).rank(axis=1, pct=True)
return (rank(mx) <= 0.8) & (rank(idio) <= 0.8)
def score(factors: list[pd.DataFrame], eligible: pd.DataFrame) -> pd.DataFrame:
return _blend(*[f.where(eligible).rank(axis=1, pct=True) for f in factors]).where(eligible)
def sleeves(p, which: list[str]) -> dict[str, pd.DataFrame]:
mem = p.member
mom = [overnight_mom(p), rec_126(p)]
out = {"c0_momentum": score(mom, mem)}
if "h4" in which:
out["h4_lottery_excluded_momentum"] = score(mom, mem & lottery_ok(p, mem))
if {"h1", "h2", "h3"} & set(which):
from research.lab import fundamentals as fu
F = fu.load_mid(p)
if "h1" in which:
out["h1_earnings_momentum"] = score([F["sue"], F["sue_rev"]], mem)
if "h2" in which:
gp = F["gp_a"].where(mem)
ok = (F["roa"] > 0) & gp.ge(gp.median(axis=1), axis=0)
out["h2_quality_screened_momentum"] = score(mom, mem & ok)
if "h3" in which:
out["h3_quality_composite"] = score([F["gp_a"], F["roa"], F["accruals"], F["net_issuance"]], mem)
return out
def sharpe(r: pd.Series) -> float:
return float(r.mean() / r.std() * np.sqrt(252))
def describe(r: pd.Series, ijh: pd.Series, book: pd.Series) -> dict:
h = r.loc[HOLDOUT[0]:HOLDOUT[1]]
e = (1 + h).cumprod()
a = alpha_stats(h, ijh)
halves = [alpha_stats(r.loc[x:y], ijh)["alpha"] for x, y in HALVES]
R = pd.DataFrame({"book": book, "s": r, "ijh": ijh}).loc[HOLDOUT[0]:HOLDOUT[1]].dropna()
with_s = monthly_blend(R, {"book": 2 / 3, "s": 1 / 3})
with_i = monthly_blend(R, {"book": 2 / 3, "ijh": 1 / 3})
ref = r.loc[REFERENCE[0]:REFERENCE[1]]
return {"cagr": float(e.iloc[-1] ** (252 / len(h)) - 1), "sharpe": sharpe(h),
"mdd": float((e / e.cummax() - 1).min()),
"alpha_ijh": float(a["alpha"]), "alpha_t": float(a["t"]), "beta_ijh": float(a["beta"]),
"alpha_halves": [float(x) for x in halves],
"corr_book": float(R["s"].corr(R["book"])),
"book_sharpe_with_sleeve": sharpe(with_s), "book_sharpe_with_ijh": sharpe(with_i),
"reference_sharpe": sharpe(ref),
"years": {int(k): float(v) for k, v in ((1 + h).groupby(h.index.year).prod() - 1).items()}}
def verdict(name: str, d: dict, c0: dict) -> dict:
a1 = d["alpha_t"] >= 2.0 or (d["alpha_t"] >= 1.5 and min(d["alpha_halves"]) > 0)
a2 = d["book_sharpe_with_sleeve"] > d["book_sharpe_with_ijh"]
checks = {"A1": a1, "A2": a2}
if name.startswith(("h2", "h4")):
checks["A3"] = d["sharpe"] > c0["sharpe"]
return {"checks": {k: bool(v) for k, v in checks.items()},
"verdict": "ACCEPT" if all(checks.values()) else "REJECT"}
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--only", nargs="+", default=["h1", "h2", "h3", "h4"])
which = ap.parse_args().only
os.makedirs(OUT, exist_ok=True)
pre = os.path.join(OUT, "prereg.json")
if not os.path.exists(pre):
with open(pre, "w") as fh:
json.dump(PREREG, fh, indent=2)
res_path = os.path.join(OUT, "results.json")
done = json.load(open(res_path)) if os.path.exists(res_path) else {}
clash = [k for k in done if k != "c0_momentum" and k.split("_")[0] in which]
if clash:
raise SystemExit(f"already run: {clash}; this study does not overwrite results")
p = load_us_mid("mid")
ijh = p.extra["etf"]["IJH"].pct_change(fill_method=None)
book = pd.read_pickle(BOOK)["base"]
S = sleeves(p, which)
span = [(REFERENCE[0], HOLDOUT[1])]
rets = {}
for name, sc in S.items():
rets[name] = run(p, sc, span, **KW)["_rets"]
print(f"ran {name}", flush=True)
ew = p.close.pct_change(fill_method=None).where(p.member.shift(1, fill_value=False)).mean(axis=1)
rets["bench_ew_members"] = ew
rets["bench_ijh"] = ijh
for name, r in rets.items():
d = describe(r.reindex(p.dates).fillna(0.0), ijh, book)
if name.startswith("h"):
d.update(verdict(name, d, done.get("c0_momentum") or describe(rets["c0_momentum"], ijh, book)))
done[name] = d
print(f"{name:32s} cagr {d['cagr']*100:5.1f}% sh {d['sharpe']:.2f} mdd {d['mdd']*100:6.1f}% "
f"alpha {d['alpha_ijh']*100:5.1f}% t {d['alpha_t']:.2f} halves "
f"{[round(x*100, 1) for x in d['alpha_halves']]} corr_book {d['corr_book']:.2f} "
f"book+sleeve {d['book_sharpe_with_sleeve']:.2f} vs book+IJH {d['book_sharpe_with_ijh']:.2f} "
f"ref {d['reference_sharpe']:.2f} {d.get('verdict', '')} {d.get('checks', '')}")
with open(res_path, "w") as fh:
json.dump(done, fh, indent=2)
pd.DataFrame(rets).to_pickle(os.path.join(OUT, f"returns_{'_'.join(which)}.pkl"))
from research.lab.trials import append_trials
append_trials(os.path.join("data", "research", "results", "trials_us.jsonl"),
[{"name": f"midcap_sleeve:{k}", **{x: y for x, y in v.items() if x != "years"}}
for k, v in done.items() if k in rets],
{"study": "midcap_sleeve", "role": "preregistered family; holdout partly seen"})
if __name__ == "__main__":
main()