diff --git a/CLAUDE.md b/CLAUDE.md index 286c1c7..329141e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,8 @@ uv run python -m research.lab.fetch_wiki_attention && PYTHONPATH=. uv run --with PYTHONPATH=. uv run --with numba python -m research.lab.concentration # industry caps on the momentum sleeve PYTHONPATH=. uv run python -m research.lab.universe_midcap # overnight_recovery_top10 on S&P 500 ∪ S&P 400 (preregistered; FAIL 2026-10-08) uv run python -m research.lab.fetch_us_fundamentals --universe mid # EDGAR PIT facts for S&P 400 ever-members (name-checked CIKs) -PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] # separate S&P 400 sleeve vs IJH (preregistered; all REJECT 2026-10-08) +uv run python -m research.lab.fetch_us_fundamentals --universe small # the same for S&P 600 (IJR) +PYTHONPATH=. uv run python -m research.lab.midcap_sleeve [--only h1..h4] [--universe small] # S&P 400 sleeves vs IJH; S&P 600 replication vs IJR (all REJECT 2026-10-08) PYTHONPATH=. uv run python -m research.lab.fetch_shortvolume # FINRA REG SHO daily short volume (~8y archive, resume-safe) PYTHONPATH=. uv run python -m research.lab.short_volume [us|midsmall|all] # SVR (short-volume ratio) IC / quintile scan on PIT panels PYTHONPATH=. uv run python -m research.lab.fetch_fails # SEC fails-to-deliver archives (2018+, resume-safe) diff --git a/research/lab/fetch_us_fundamentals.py b/research/lab/fetch_us_fundamentals.py index 2b20b2d..cb9cd53 100644 --- a/research/lab/fetch_us_fundamentals.py +++ b/research/lab/fetch_us_fundamentals.py @@ -5,8 +5,9 @@ history (needed to turn split-adjusted prices back into market caps). uv run python -m research.lab.fetch_us_fundamentals uv run python -m research.lab.fetch_us_fundamentals --universe mid # S&P 400 (IJH) ever-members + uv run python -m research.lab.fetch_us_fundamentals --universe small # S&P 600 (IJR) ever-members -Writes data/research/us_fund_raw.pkl (us_mid_fund_raw.pkl for --universe mid): +Writes data/research/us_fund_raw.pkl (us_{mid,small}_fund_raw.pkl for --universe): {"facts": DataFrame[ticker, tag, start, end, val, filed, form], "splits": DataFrame (dates x tickers, split ratio on split days)} @@ -27,7 +28,7 @@ import yfinance as yf from research.lab.fetch_us import _proxy_env OUT = os.path.join("data", "research", "us_fund_raw.pkl") -MID_OUT = os.path.join("data", "research", "us_mid_fund_raw.pkl") +UNIVERSE_FUND = {"mid": "IJH", "small": "IJR"} # --universe -> iShares fund UA = {"User-Agent": "quant-research/0.1 gahow@example.com", "Accept-Encoding": "gzip"} FORMS = {"10-K", "10-Q", "10-K/A", "10-Q/A", "10-KT", "20-F", "40-F"} @@ -148,8 +149,8 @@ def _name_tokens(name: str) -> set[str]: return {w for w in words if w not in _SUFFIXES and len(w) > 1} -def mid_ciks(tickers: list[str]) -> tuple[dict[str, int], pd.DataFrame]: - """CIKs for S&P 400 tickers. Candidates come from sec_sic.csv and +def mid_ciks(tickers: list[str], fund: str = "IJH") -> tuple[dict[str, int], pd.DataFrame]: + """CIKs for S&P 400 (IJH) or S&P 600 (IJR) tickers. Candidates come from sec_sic.csv and mid_sec_sic.csv (earlier runs), then today's company_tickers.json; both map a ticker to whoever holds it NOW, so a candidate counts only if one of its SEC names (current or former) shares a word with an iShares holding @@ -177,7 +178,7 @@ def mid_ciks(tickers: list[str]) -> tuple[dict[str, int], pd.DataFrame]: js = r.json() names[str(c)] = [js.get("name", "")] + [x.get("name", "") for x in js.get("formerNames", [])] hold = load_holdings() - hnames = hold[hold["fund"] == "IJH"].groupby("ticker")["name"].apply(lambda s: set(s.dropna())) + hnames = hold[hold["fund"] == fund].groupby("ticker")["name"].apply(lambda s: set(s.dropna())) rows, ok = [], {} for t in tickers: c = cand.get(t, 0) @@ -201,13 +202,14 @@ def mid_ciks(tickers: list[str]) -> tuple[dict[str, int], pd.DataFrame]: def main() -> None: import sys global OUT - if "--universe" in sys.argv and sys.argv[sys.argv.index("--universe") + 1] == "mid": + uni = sys.argv[sys.argv.index("--universe") + 1] if "--universe" in sys.argv else None + if uni in UNIVERSE_FUND: from research.lab.panel_mid import load_us_mid - OUT = MID_OUT - tickers = list(load_us_mid("mid").close.columns) - ciks, audit = mid_ciks(tickers) - audit.to_csv(os.path.join("data", "research", "us_mid_fund_cik_audit.csv"), index=False) - print(f"{len(ciks)}/{len(tickers)} S&P 400 tickers have a name-checked CIK", flush=True) + OUT = os.path.join("data", "research", f"us_{uni}_fund_raw.pkl") + tickers = list(load_us_mid(uni).close.columns) + ciks, audit = mid_ciks(tickers, UNIVERSE_FUND[uni]) + audit.to_csv(os.path.join("data", "research", f"us_{uni}_fund_cik_audit.csv"), index=False) + print(f"{len(ciks)}/{len(tickers)} {UNIVERSE_FUND[uni]} tickers have a name-checked CIK", flush=True) else: with open(os.path.join("data", "research", "us_ohlcv.pkl"), "rb") as fh: tickers = [t for t in pickle.load(fh)["close"].columns] diff --git a/research/lab/fundamentals.py b/research/lab/fundamentals.py index c03b6ea..60ac8a8 100644 --- a/research/lab/fundamentals.py +++ b/research/lab/fundamentals.py @@ -294,6 +294,10 @@ def load(p, rebuild: bool = False, raw_path: str = RAW, cache: str = CACHE) -> d return F -def load_mid(p, rebuild: bool = False) -> dict[str, pd.DataFrame]: - """The same factors for the S&P 400 panel (research/lab/panel_mid.py).""" - return load(p, rebuild, MID_RAW, MID_CACHE) +def load_mid(p, rebuild: bool = False, which: str = "mid") -> dict[str, pd.DataFrame]: + """The same factors for the S&P 400 ("mid") or S&P 600 ("small") panel + (research/lab/panel_mid.py; raw facts from fetch_us_fundamentals --universe).""" + if which == "mid": + return load(p, rebuild, MID_RAW, MID_CACHE) + return load(p, rebuild, os.path.join("data", "research", f"us_{which}_fund_raw.pkl"), + os.path.join("data", "research", f"us_{which}_fund_factors.pkl")) diff --git a/research/lab/midcap_sleeve.py b/research/lab/midcap_sleeve.py index 23a2f9e..77aebcd 100644 --- a/research/lab/midcap_sleeve.py +++ b/research/lab/midcap_sleeve.py @@ -4,6 +4,7 @@ alpha over IJH and improve the book more than holding IJH would? PYTHONPATH=. uv run python -m research.lab.midcap_sleeve --only h4 PYTHONPATH=. uv run python -m research.lab.midcap_sleeve --only h1 h2 h3 # needs us_mid_fund_raw.pkl + PYTHONPATH=. uv run python -m research.lab.midcap_sleeve --universe small # S&P 600 replication (REPLICATION) Background (research/lab/universe_midcap.py): ranking S&P 400 names inside the large-cap momentum top 10 lost (Sharpe 0.86 vs 0.98, 2019-26); the S&P @@ -42,11 +43,22 @@ size. Boundary: gp_a covers 55-70% of members (no gross profit for banks, REITs), so h2's median screen also drops most financials. EDGAR CIKs are name-checked against iShares holding names (723/845 tickers; members covered 84-98% a year from 2017). + +S&P 600 replication (2026-10-08, REPLICATION written before the S&P 600 +fundamentals were fetched; 963/1112 tickers, members covered 82-99% a year +from 2017): the quality result does not replicate. h3 alpha vs IJR +2.3%/yr +(t 0.78), halves +8.0 / -3.5; h2 +2.2% (t 0.73), halves +5.7 / -1.7; h1 ++2.7% (t 0.90); h4 +2.2% (t 0.94); all REJECT, and weaker at 40 bp. The +equal-weight member basket beats IJR (alpha +2.8%, t 3.2, Sharpe 0.60) and +every sleeve on Sharpe. Per the rule (h3 alpha <= 0 in a half), quality +stock selection in mid/small caps is dropped. In both universes the +selection signals earn their alpha in 2017-21 and lose it in 2022-26. """ from __future__ import annotations import argparse +import dataclasses import json import os import warnings @@ -55,6 +67,7 @@ import numpy as np import pandas as pd from research.lab.alpha import alpha_stats +from research.lab.engine import Costs from research.lab.factors import overnight_mom, rec_126 from research.lab.panel_mid import load_us_mid from research.lab.us_final import monthly_blend @@ -96,6 +109,24 @@ PREREG = { "reference_only": f"{REFERENCE}: price coverage 57-78%, survivor-biased", "status": "holdout overlaps periods seen in earlier mid/small scans; not independent OOS", } +REPLICATION = { + "question": "Do the S&P 400 quality sleeves (h3 primary, h2 secondary) replicate on S&P 600?", + "written": "2026-10-08, after the S&P 400 results (h3 +3.4%/yr t 1.23, h2 +3.5% t 1.28) and before any " + "S&P 600 fundamentals were fetched", + "definitions": "unchanged from PREREG (signals, top 30, every 21, buffer 1, 21 phases); universe = PIT S&P 600 " + "members (IJR holdings), benchmark IJR; c0, h1, h4 reported, not judged", + "accept_if_all": [ + "A1: alpha vs IJR over 2017-01-03..2026-09-24, Newey-West t >= 2.0, or t >= 1.5 and alpha > 0 in both halves", + "A2: Sharpe(2/3 book + 1/3 sleeve) > Sharpe(2/3 book + 1/3 IJR)", + "A3 (h2): Sharpe(h2) > Sharpe(c0_momentum)", + ], + "costs": "20 bp a side decides; 40 bp a side reported (small-cap spreads)", + "decision": { + "h3 accepted": "propose a forward-only S&P 400+600 quality sleeve in a validation book", + "h3 rejected, alpha > 0 in both halves": "consistent but unproven; no action, no further tuning on 2017-26", + "h3 alpha <= 0 in either half": "drop quality stock selection in mid/small caps", + }, +} def lottery_ok(p, members: pd.DataFrame) -> pd.DataFrame: @@ -116,7 +147,7 @@ def score(factors: list[pd.DataFrame], eligible: pd.DataFrame) -> pd.DataFrame: return _blend(*[f.where(eligible).rank(axis=1, pct=True) for f in factors]).where(eligible) -def sleeves(p, which: list[str]) -> dict[str, pd.DataFrame]: +def sleeves(p, which: list[str], universe: str = "mid") -> dict[str, pd.DataFrame]: mem = p.member mom = [overnight_mom(p), rec_126(p)] out = {"c0_momentum": score(mom, mem)} @@ -124,7 +155,7 @@ def sleeves(p, which: list[str]) -> dict[str, pd.DataFrame]: out["h4_lottery_excluded_momentum"] = score(mom, mem & lottery_ok(p, mem)) if {"h1", "h2", "h3"} & set(which): from research.lab import fundamentals as fu - F = fu.load_mid(p) + F = fu.load_mid(p, which=universe) if "h1" in which: out["h1_earnings_momentum"] = score([F["sue"], F["sue_rev"]], mem) if "h2" in which: @@ -141,6 +172,7 @@ def sharpe(r: pd.Series) -> float: def describe(r: pd.Series, ijh: pd.Series, book: pd.Series) -> dict: + """Holdout stats; ``ijh`` is the universe's ETF (IJH or IJR).""" h = r.loc[HOLDOUT[0]:HOLDOUT[1]] e = (1 + h).cumprod() a = alpha_stats(h, ijh) @@ -172,48 +204,57 @@ def verdict(name: str, d: dict, c0: dict) -> dict: def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--only", nargs="+", default=["h1", "h2", "h3", "h4"]) - which = ap.parse_args().only - os.makedirs(OUT, exist_ok=True) - pre = os.path.join(OUT, "prereg.json") + ap.add_argument("--universe", choices=["mid", "small"], default="mid") + args = ap.parse_args() + which, uni = args.only, args.universe + out = OUT if uni == "mid" else OUT + "_sp600" + etf = "IJH" if uni == "mid" else "IJR" + os.makedirs(out, exist_ok=True) + pre = os.path.join(out, "prereg.json") if not os.path.exists(pre): with open(pre, "w") as fh: - json.dump(PREREG, fh, indent=2) - res_path = os.path.join(OUT, "results.json") + json.dump(PREREG if uni == "mid" else {**REPLICATION, "base": PREREG}, fh, indent=2) + res_path = os.path.join(out, "results.json") done = json.load(open(res_path)) if os.path.exists(res_path) else {} clash = [k for k in done if k != "c0_momentum" and k.split("_")[0] in which] if clash: raise SystemExit(f"already run: {clash}; this study does not overwrite results") - p = load_us_mid("mid") - ijh = p.extra["etf"]["IJH"].pct_change(fill_method=None) + p = load_us_mid(uni) + bench = p.extra["etf"][etf].pct_change(fill_method=None) book = pd.read_pickle(BOOK)["base"] - S = sleeves(p, which) + S = sleeves(p, which, uni) span = [(REFERENCE[0], HOLDOUT[1])] + costs = {"": p.costs} if uni == "mid" else {"": p.costs, "@40bp": Costs(buy=0.004, sell=0.004)} rets = {} - for name, sc in S.items(): - rets[name] = run(p, sc, span, **KW)["_rets"] - print(f"ran {name}", flush=True) + for tag, c in costs.items(): + pc = dataclasses.replace(p, costs=c) + for name, sc in S.items(): + rets[name + tag] = run(pc, sc, span, **KW)["_rets"] + print(f"ran {name}{tag}", flush=True) ew = p.close.pct_change(fill_method=None).where(p.member.shift(1, fill_value=False)).mean(axis=1) rets["bench_ew_members"] = ew - rets["bench_ijh"] = ijh + rets[f"bench_{etf.lower()}"] = bench for name, r in rets.items(): - d = describe(r.reindex(p.dates).fillna(0.0), ijh, book) + d = describe(r.reindex(p.dates).fillna(0.0), bench, book) if name.startswith("h"): - d.update(verdict(name, d, done.get("c0_momentum") or describe(rets["c0_momentum"], ijh, book))) + ctrl = "c0_momentum" + ("@40bp" if name.endswith("@40bp") else "") + d.update(verdict(name, d, done.get(ctrl) or describe(rets[ctrl], bench, book))) done[name] = d - print(f"{name:32s} cagr {d['cagr']*100:5.1f}% sh {d['sharpe']:.2f} mdd {d['mdd']*100:6.1f}% " + print(f"{name:38s} cagr {d['cagr']*100:5.1f}% sh {d['sharpe']:.2f} mdd {d['mdd']*100:6.1f}% " f"alpha {d['alpha_ijh']*100:5.1f}% t {d['alpha_t']:.2f} halves " f"{[round(x*100, 1) for x in d['alpha_halves']]} corr_book {d['corr_book']:.2f} " - f"book+sleeve {d['book_sharpe_with_sleeve']:.2f} vs book+IJH {d['book_sharpe_with_ijh']:.2f} " + f"book+sleeve {d['book_sharpe_with_sleeve']:.2f} vs book+{etf} {d['book_sharpe_with_ijh']:.2f} " f"ref {d['reference_sharpe']:.2f} {d.get('verdict', '')} {d.get('checks', '')}") with open(res_path, "w") as fh: json.dump(done, fh, indent=2) - pd.DataFrame(rets).to_pickle(os.path.join(OUT, f"returns_{'_'.join(which)}.pkl")) + pd.DataFrame(rets).to_pickle(os.path.join(out, f"returns_{'_'.join(which)}.pkl")) from research.lab.trials import append_trials append_trials(os.path.join("data", "research", "results", "trials_us.jsonl"), - [{"name": f"midcap_sleeve:{k}", **{x: y for x, y in v.items() if x != "years"}} + [{"name": f"midcap_sleeve[{uni}]:{k}", **{x: y for x, y in v.items() if x != "years"}} for k, v in done.items() if k in rets], - {"study": "midcap_sleeve", "role": "preregistered family; holdout partly seen"}) + {"study": "midcap_sleeve" if uni == "mid" else "midcap_sleeve_sp600_replication", + "role": "preregistered; holdout partly seen"}) if __name__ == "__main__":