research: factor census on the current panel; key the factor-rank cache to the panel
factor_census measures all 86 lab factors one way on the post-Tiingo PIT S&P 500 panel (IC, quintile spread, top-20 alpha vs EW). One class A (low_svr21, 2018+), two class B (accruals, rev_5), 69 N, no |t| >= 3. Momentum and overnight_mom have no quintile spread; the house edge sits in the top 10. --part construction compares the live construction. rotation.factor_ranks reused ranks_us.pkl regardless of the panel. The cache predated the Tiingo merge (643 of 787 columns), so callers silently dropped the backfilled delisted names. The cache now carries a key file (panel calendar, tickers, membership, factors.py hash) and is rebuilt when it does not match. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,8 @@ through the same engine (PIT members, next-open fills, phase averaging).
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import time
|
||||
@@ -32,13 +34,38 @@ warnings.filterwarnings("ignore")
|
||||
# inputs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def ranks_signature(p) -> dict:
|
||||
"""What the cached ranks depend on: the panel's calendar, tickers and
|
||||
membership, and the factor definitions (factors.py as a whole, so a
|
||||
changed helper invalidates every factor)."""
|
||||
with open(os.path.join(os.path.dirname(__file__), "factors.py"), "rb") as fh:
|
||||
src = hashlib.sha256(fh.read()).hexdigest()[:16]
|
||||
cols = hashlib.sha256("\n".join(map(str, p.close.columns)).encode()).hexdigest()[:16]
|
||||
return {"market": p.market, "dates": [str(p.dates[0].date()), str(p.dates[-1].date()), len(p.dates)],
|
||||
"columns": [len(p.close.columns), cols], "member_days": int(p.member.to_numpy().sum()),
|
||||
"factors_py": src}
|
||||
|
||||
|
||||
def factor_ranks(p, market: str, names: list[str] | None = None) -> dict[str, pd.DataFrame]:
|
||||
"""Percentile ranks among PIT members, cached per market (float32)."""
|
||||
"""Percentile ranks among PIT members, cached per market (float32).
|
||||
|
||||
The cache is used only while ranks_{market}.key matches the panel and
|
||||
factors.py; otherwise it is rebuilt (a stale cache silently dropped the
|
||||
Tiingo-backfilled names)."""
|
||||
path = os.path.join("data", "research", f"ranks_{market}.pkl")
|
||||
key = os.path.join("data", "research", f"ranks_{market}.key")
|
||||
sig = ranks_signature(p)
|
||||
cache = {}
|
||||
if os.path.exists(path):
|
||||
with open(path, "rb") as fh:
|
||||
cache = pickle.load(fh)
|
||||
if os.path.exists(path) and os.path.exists(key):
|
||||
with open(key) as fh:
|
||||
fresh = json.load(fh) == sig
|
||||
if fresh:
|
||||
with open(path, "rb") as fh:
|
||||
cache = pickle.load(fh)
|
||||
else:
|
||||
print(f"ranks_{market}.pkl is stale for this panel or factors.py; rebuilding", flush=True)
|
||||
elif os.path.exists(path):
|
||||
print(f"ranks_{market}.pkl has no key file; rebuilding", flush=True)
|
||||
mk = "us" if market == "us" else "cn"
|
||||
names = names or [n for n, f in FACTORS.items() if mk in f.markets]
|
||||
dirty = False
|
||||
@@ -57,6 +84,8 @@ def factor_ranks(p, market: str, names: list[str] | None = None) -> dict[str, pd
|
||||
if dirty:
|
||||
with open(path, "wb") as fh:
|
||||
pickle.dump(cache, fh, protocol=4)
|
||||
with open(key, "w") as fh:
|
||||
json.dump(sig, fh)
|
||||
return {n: cache[n] for n in names if n in cache}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user