research: factor census on the current panel; key the factor-rank cache to the panel

factor_census measures all 86 lab factors one way on the post-Tiingo PIT
S&P 500 panel (IC, quintile spread, top-20 alpha vs EW). One class A
(low_svr21, 2018+), two class B (accruals, rev_5), 69 N, no |t| >= 3.
Momentum and overnight_mom have no quintile spread; the house edge sits
in the top 10. --part construction compares the live construction.

rotation.factor_ranks reused ranks_us.pkl regardless of the panel. The
cache predated the Tiingo merge (643 of 787 columns), so callers silently
dropped the backfilled delisted names. The cache now carries a key file
(panel calendar, tickers, membership, factors.py hash) and is rebuilt
when it does not match.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-12 07:14:42 -07:00
parent ed0ed3f4b2
commit 81f4e3b32c
3 changed files with 396 additions and 4 deletions

View File

@@ -13,6 +13,8 @@ through the same engine (PIT members, next-open fills, phase averaging).
from __future__ import annotations
import argparse
import hashlib
import json
import os
import pickle
import time
@@ -32,13 +34,38 @@ warnings.filterwarnings("ignore")
# inputs
# ---------------------------------------------------------------------------
def ranks_signature(p) -> dict:
"""What the cached ranks depend on: the panel's calendar, tickers and
membership, and the factor definitions (factors.py as a whole, so a
changed helper invalidates every factor)."""
with open(os.path.join(os.path.dirname(__file__), "factors.py"), "rb") as fh:
src = hashlib.sha256(fh.read()).hexdigest()[:16]
cols = hashlib.sha256("\n".join(map(str, p.close.columns)).encode()).hexdigest()[:16]
return {"market": p.market, "dates": [str(p.dates[0].date()), str(p.dates[-1].date()), len(p.dates)],
"columns": [len(p.close.columns), cols], "member_days": int(p.member.to_numpy().sum()),
"factors_py": src}
def factor_ranks(p, market: str, names: list[str] | None = None) -> dict[str, pd.DataFrame]:
"""Percentile ranks among PIT members, cached per market (float32)."""
"""Percentile ranks among PIT members, cached per market (float32).
The cache is used only while ranks_{market}.key matches the panel and
factors.py; otherwise it is rebuilt (a stale cache silently dropped the
Tiingo-backfilled names)."""
path = os.path.join("data", "research", f"ranks_{market}.pkl")
key = os.path.join("data", "research", f"ranks_{market}.key")
sig = ranks_signature(p)
cache = {}
if os.path.exists(path):
with open(path, "rb") as fh:
cache = pickle.load(fh)
if os.path.exists(path) and os.path.exists(key):
with open(key) as fh:
fresh = json.load(fh) == sig
if fresh:
with open(path, "rb") as fh:
cache = pickle.load(fh)
else:
print(f"ranks_{market}.pkl is stale for this panel or factors.py; rebuilding", flush=True)
elif os.path.exists(path):
print(f"ranks_{market}.pkl has no key file; rebuilding", flush=True)
mk = "us" if market == "us" else "cn"
names = names or [n for n, f in FACTORS.items() if mk in f.markets]
dirty = False
@@ -57,6 +84,8 @@ def factor_ranks(p, market: str, names: list[str] | None = None) -> dict[str, pd
if dirty:
with open(path, "wb") as fh:
pickle.dump(cache, fh, protocol=4)
with open(key, "w") as fh:
json.dump(sig, fh)
return {n: cache[n] for n in names if n in cache}