Gahow Wang
04e2c5c3be
Studies run by earlier sessions and documented in CLAUDE.md, but never committed: FINRA short volume (SVR) and the research-only svr_momentum veto, SEC fails-to-deliver, 13D activists and the 13D event sleeve, 13F institutional flow, GDELT news volume, S&P 400/600 Form 4 insiders, union_shield, shield_lab; plus the mid/small insider and Tiingo fetcher changes and midsmall_rebuild. overnight_midsmall: the overnight-only momentum book on S&P 400/600 earns 0.93 net Sharpe at 3 bp a side (2019-26) with corr 0.83 to the S&P 500 book (STOP). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
71 lines
2.6 KiB
Python
71 lines
2.6 KiB
Python
"""Convert mid/small Tiingo caches, rebuild the PIT panels, report the
|
|
survivorship change (pre-Tiingo backup and the previous build vs rebuilt
|
|
member counts per year). Converts every cached ticker on the mid/small plan
|
|
(plus the batch list), so it is re-runnable after each monthly Tiingo batch.
|
|
|
|
PYTHONPATH=. uv run python -m research.midsmall_rebuild
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import pickle
|
|
|
|
import pandas as pd
|
|
|
|
|
|
def convert_list() -> list[str]:
|
|
plan = json.load(open("data/research/tiingo_plan.json"))
|
|
txt = [l.strip() for l in open("data/research/tiingo_midsmall.txt") if l.strip()]
|
|
want = set(plan["mid_small"]) | set(txt)
|
|
cached = [f[:-4] for f in os.listdir("data/research/tiingo") if f.endswith(".csv")]
|
|
return sorted(set(cached) & want)
|
|
|
|
|
|
def main() -> None:
|
|
conv = convert_list()
|
|
print(f"mid/small cached tickers to convert: {len(conv)}", flush=True)
|
|
|
|
from research.lab.tiingo_merge import write_yahoo_mid
|
|
write_yahoo_mid(conv)
|
|
|
|
prev_counts = None
|
|
pkl = "data/research/us_midsmall.panel.pkl"
|
|
if os.path.exists(pkl):
|
|
with open(pkl, "rb") as fh:
|
|
prev = pickle.load(fh)
|
|
prev_counts = prev.member.sum(axis=1).resample("YE").last()
|
|
del prev
|
|
|
|
for f in ("us_mid_bars.pkl", "us_mid.panel.pkl", "us_small.panel.pkl", "us_midsmall.panel.pkl"):
|
|
p = os.path.join("data", "research", f)
|
|
if os.path.exists(p):
|
|
os.remove(p)
|
|
print(f"removed {p}", flush=True)
|
|
|
|
from research.lab.panel_mid import load_us_mid
|
|
new = load_us_mid("midsmall")
|
|
load_us_mid("mid")
|
|
load_us_mid("small")
|
|
with open("data/research/us_midsmall.panel.pre_tiingo.pkl", "rb") as fh:
|
|
old = pickle.load(fh)
|
|
|
|
n, o = new.member.sum(axis=1), old.member.sum(axis=1)
|
|
comp = pd.DataFrame({"post": n.resample("YE").last(), "pre_tiingo": o.resample("YE").last(),
|
|
"prev_build": prev_counts}).dropna(subset=["pre_tiingo"])
|
|
comp["delta_prev"] = comp["post"] - comp["prev_build"]
|
|
comp.index = comp.index.year
|
|
print("\nPIT members per year (pre_tiingo -> previous build -> post):")
|
|
print(comp.to_string())
|
|
from research.lab.fetch_us_mid import load_bars
|
|
restored = [t for t in conv if t in new.close.columns
|
|
and new.member[t].sum() > 0 and load_bars(t) is not None
|
|
and t not in old.close.columns]
|
|
print(f"\nnew PIT-covered names not in old panel: {len(restored)}")
|
|
print(new.close.shape, flush=True)
|
|
print("REBUILD DONE", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|