gahow/quant-trading
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: commit the 2026-10-07/08 studies left in the working tree, overnight mid/small (STOP)

Browse Source 
Studies run by earlier sessions and documented in CLAUDE.md, but never
committed: FINRA short volume (SVR) and the research-only svr_momentum veto,
SEC fails-to-deliver, 13D activists and the 13D event sleeve, 13F
institutional flow, GDELT news volume, S&P 400/600 Form 4 insiders,
union_shield, shield_lab; plus the mid/small insider and Tiingo fetcher
changes and midsmall_rebuild.

overnight_midsmall: the overnight-only momentum book on S&P 400/600 earns
0.93 net Sharpe at 3 bp a side (2019-26) with corr 0.83 to the S&P 500 book
(STOP).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-10-09 22:20:43 -07:00
parent 36cb125719
commit 04e2c5c3be
21 changed files with 2543 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
research/lab/fetch_shortvolume.py Normal file
View File

@@ -0,0 +1,148 @@
"""Fetch FINRA REG SHO daily short-sale volume for the research universes.
One public file per trading day covers every NMS symbol (no auth):
https://cdn.finra.org/equity/regsho/daily/CNMSshvolYYYYMMDD.txt
Date|Symbol|ShortVolume|ShortExemptVolume|TotalVolume|Market
Writes data/research/short_volume.pkl: {"short": DataFrame, "total": DataFrame}
(float32, date x symbol), filtered to the symbols of data/us.csv plus the US
mid/small panels (us_mid / us_midsmall). Resume-safe: re-running fetches only
days that are missing from the store. Network goes through QUANT_MARKET_PROXY
when set (same convention as the other fetchers).
PYTHONPATH=. uv run python -m research.lab.fetch_shortvolume [--days-limit N]
"""
from __future__ import annotations
import json
import os
import pickle
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
import pandas as pd
import requests
from research.lab.fetch_us import _proxy_env
OUT = os.path.join("data", "research", "short_volume.pkl")
SYMBOLS_CACHE = os.path.join("data", "research", "shortvolume_symbols.json")
URL = "https://cdn.finra.org/equity/regsho/daily/CNMSshvol{day}.txt"
HEADERS = {"User-Agent": "Mozilla/5.0 (research; contact local)"}
WORKERS = 6
def universe_symbols() -> list[str]:
if os.path.exists(SYMBOLS_CACHE):
return json.load(open(SYMBOLS_CACHE))
syms = set()
us = pd.read_csv("data/us.csv", index_col=0, nrows=0)
syms.update(us.columns)
for path in ("data/research/us_mid.panel.pkl", "data/research/us_midsmall.panel.pkl"):
if os.path.exists(path):
with open(path, "rb") as fh:
panel = pickle.load(fh)
cols = getattr(panel, "close").columns if hasattr(panel, "close") else None
if cols is None and isinstance(panel, dict):
cols = panel["close"].columns
syms.update(str(c).replace(".", "-") for c in cols)
syms.discard("")
out = sorted(syms)
json.dump(out, open(SYMBOLS_CACHE, "w"))
return out
def trading_days() -> list[str]:
close = pd.read_csv("data/us.csv", index_col=0, usecols=[0], parse_dates=True)
days = close.index
if os.path.exists("data/etfs.csv"):
etfs = pd.read_csv("data/etfs.csv", index_col=0, usecols=[0], parse_dates=True)
days = days.union(etfs.index)
return [d.strftime("%Y%m%d") for d in days]
def fetch_day(day: str, symbols: set[str]) -> tuple[str, dict, dict] | None:
url = URL.format(day=day)
for attempt in range(4):
try:
r = requests.get(url, headers=HEADERS, timeout=40)
if r.status_code in (403, 404):
# FINRA's public archive keeps only ~8 years; older days are gone.
return day, {}, {}
r.raise_for_status()
short, total = {}, {}
for line in r.text.splitlines()[1:]:
parts = line.split("|")
if len(parts) < 5:
continue
sym = parts[1]
if sym not in symbols:
continue
try:
s, t = float(parts[2]), float(parts[4])
except ValueError:
continue
short[sym] = short.get(sym, 0.0) + s
total[sym] = total.get(sym, 0.0) + t
return day, short, total
except Exception as exc: # noqa: BLE001
if attempt == 3:
print(f" FAILED {day}: {exc}")
else:
import time
time.sleep(1.5 * (attempt + 1))
return None
def main() -> None:
_proxy_env()
limit = None
if "--days-limit" in sys.argv:
limit = int(sys.argv[sys.argv.index("--days-limit") + 1])
symbols = set(universe_symbols())
print(f"universe symbols: {len(symbols)}")
days = trading_days()
prev_short = prev_total = None
have: set[str] = set()
if os.path.exists(OUT):
with open(OUT, "rb") as fh:
d = pickle.load(fh)
prev_short, prev_total = d["short"], d["total"]
have = {pd.Timestamp(i).strftime("%Y%m%d") for i in prev_short.index}
todo = [d for d in days if d not in have]
if limit:
todo = todo[:limit]
print(f"days: {len(days)}, already stored: {len(have)}, fetching: {len(todo)}")
short_rows: dict[str, dict] = {}
total_rows: dict[str, dict] = {}
with ThreadPoolExecutor(max_workers=WORKERS) as ex:
futs = {ex.submit(fetch_day, d, symbols): d for d in todo}
done = 0
for fut in as_completed(futs):
res = fut.result()
done += 1
if res is not None:
day, short, total = res
short_rows[day] = short
total_rows[day] = total
if done % 200 == 0:
print(f" {done}/{len(todo)} fetched")
new_short = pd.DataFrame.from_dict(short_rows, orient="index")
new_total = pd.DataFrame.from_dict(total_rows, orient="index")
for df in (new_short, new_total):
df.index = pd.to_datetime(df.index, format="%Y%m%d")
frames_short = [f for f in (prev_short, new_short) if f is not None and len(f)]
frames_total = [f for f in (prev_total, new_total) if f is not None and len(f)]
df_short = pd.concat(frames_short).sort_index()
df_total = pd.concat(frames_total).sort_index()
with open(OUT, "wb") as fh:
pickle.dump({"short": df_short.astype("float32"), "total": df_total.astype("float32")}, fh)
print(f"stored {df_short.shape} short / {df_total.shape} total -> {OUT}")
if __name__ == "__main__":
main()