Gahow Wang
ad569f1cea
Register us_momentum_qqq_trend: half overnight_recovery_top10, half a QQQ trend sleeve (strategies/qqq_trend.py: 200-day switch times a 30% vol target, held as TQQQ + SHY). Mixed stock+ETF, US only (extra tickers QQQ, SHY, TQQQ). Research (research/lab/): alpha against QQQ with a 2006-16 design / 2017-26 holdout split (alpha.py, qqq_scan.py, beat_qqq.py); leverage and drawdown frontier (levered_book.py, us_target50.py); synthetic 3x ETFs from 1990 and TrendRider/QQQ-trend stress tests (fetch_long_history.py, letf_history.py, vol_trend.py); short-horizon sleeves, diversifiers, industry momentum, earnings-announcement returns; new point-in-time data sources: SEC Form 4 insider trades (fetch_insider.py, insider.py), 10-K/10-Q text similarity, 8-K earnings dates and SIC codes (fetch_lazy_prices.py, lazy_prices.py), and an S&P 400/600 universe from iShares holdings (fetch_us_mid.py, panel_mid.py). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
463 lines
21 KiB
Python
463 lines
21 KiB
Python
"""
|
|
Point-in-time S&P MidCap 400 / SmallCap 600 research universe: month-end
|
|
holdings of the iShares Core ETFs that track them (IJH, IJR), and Yahoo daily
|
|
bars for every ticker that was ever held. research/lab/panel_mid.py builds
|
|
the Panels.
|
|
|
|
PYTHONPATH=. uv run python -m research.lab.fetch_us_mid holdings # ~45 min; then parses
|
|
PYTHONPATH=. uv run python -m research.lab.fetch_us_mid prices # ~3 h; resumable
|
|
PYTHONPATH=. uv run python -m research.lab.fetch_us_mid parse # re-parse cached CSVs
|
|
PYTHONPATH=. uv run python -m research.lab.fetch_us_mid status # counts, weight sums
|
|
|
|
Holdings. BlackRock's product-page API serves past month-end snapshots
|
|
(``get-fund-document?...&asOfDate=YYYYMMDD&component=holdings``; only the
|
|
last trading day of a month works) from 2006-09-29 for both funds, except
|
|
2017-01..2017-06, which return nothing. Raw CSVs are cached in
|
|
data/research/ext/ishares/<FUND>/<date>.csv; months that return nothing are
|
|
remembered in index.json. Requests go out directly (no proxy), one per ~1.2 s.
|
|
|
|
Old snapshots list each security under its LAST ticker in BlackRock's
|
|
security master (Westwood One is "DIAL" in 2006), with internal codes
|
|
(``WM1`` is Williams-Sonoma, ``VRX.1``, ``PSYS.``), no weights (quantity and
|
|
price only), and a few rows without a ticker. ``_resolve`` maps codes and
|
|
missing tickers to the clean symbol the company name carries in other
|
|
snapshots, else strips the suffix; panel_mid's price check catches what is
|
|
still wrong. Rights, warrants and stub positions (under 2% of an equal
|
|
weight: shares left over after a deletion) are dropped and weights are
|
|
recomputed from quantity x price. Outputs: holdings.csv (one row per fund,
|
|
date, ticker) and snapshots.csv (count, equity value, shares outstanding).
|
|
|
|
Prices. Yahoo daily bars from 2005 (auto_adjust=False so the as-traded close
|
|
can be checked against the holdings file's price), one ticker per request,
|
|
cached in data/research/ext/yahoo_mid/<ticker>.csv; tickers Yahoo does not
|
|
serve (mostly delisted names) are listed in missing.json. The connection is
|
|
the shell's default one (Yahoo blocks this host's own IP); the proxy in
|
|
QUANT_MARKET_PROXY is removed from the environment, because the live monitor
|
|
fetches through it. Pauses 1.5 s between tickers, 60 s after an error, and
|
|
10 minutes after five errors in a row; a run of 25 misses is re-checked
|
|
against SPY in case Yahoo is throttling.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
import urllib.request
|
|
from datetime import date, datetime, timedelta
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
ROOT = os.path.join("data", "research", "ext")
|
|
HOLD_DIR = os.path.join(ROOT, "ishares")
|
|
PX_DIR = os.path.join(ROOT, "yahoo_mid")
|
|
HOLDINGS = os.path.join(HOLD_DIR, "holdings.csv")
|
|
FUNDS = {"IJH": 239763, "IJR": 239774} # S&P MidCap 400, S&P SmallCap 600
|
|
ETFS = ["IJH", "IJR", "MDY", "SLY", "IWM", "SPY", "QQQ", "BIL", "SHY"]
|
|
API = ("https://www.ishares.com/varnish-api/blk-one01-product-data/product-data/api/"
|
|
"v1/get-fund-document?appType=PRODUCT_PAGE&appSubType=ISHARES&targetSite=us-ishares"
|
|
"&locale=en_US&portfolioId={pid}&userType=individual&asOfDate={d}&component=holdings")
|
|
DATES_API = ("https://www.ishares.com/varnish-api/blk-one01-product-data/product-data/api/"
|
|
"v2/get-product-data?appSubType=ISHARES&appType=PRODUCT_PAGE&component=holdings"
|
|
"&locale=en_US&portfolioId={pid}&targetSite=us-ishares&userType=individual"
|
|
"&excludeContent=true&includeConfig=true")
|
|
UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"
|
|
FIRST_MONTH = "2006-01"
|
|
NOT_STOCK = r"\bRIGHTS?\b|\bRTS\b|WARRANT|\bWTS?\b|\bCVR\b|ESCROW|CONTRA\b|CASH ACCRUAL|VESTING"
|
|
STUB_REL = 0.02 # positions below 2% of an equal weight are leftovers
|
|
CASH_SECTORS = {"S-T Securities", "Cash and/or Derivatives", "Cash", "Money Market"}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# holdings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_direct = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
|
|
|
|
def _get(url: str, headers: dict | None = None) -> bytes:
|
|
req = urllib.request.Request(url, headers={"User-Agent": UA, **(headers or {})})
|
|
for attempt in range(4):
|
|
try:
|
|
with _direct.open(req, timeout=60) as fh:
|
|
return fh.read()
|
|
except Exception as exc: # noqa: BLE001 — retry transient network errors
|
|
print(f" {type(exc).__name__}: {exc}; retry {attempt + 1}", flush=True)
|
|
time.sleep(10 * (attempt + 1))
|
|
raise RuntimeError(f"giving up on {url}")
|
|
|
|
|
|
def _month_end_candidates(month: pd.Period) -> list[date]:
|
|
"""The last four weekdays of a month, latest first (the last trading day
|
|
is one of them)."""
|
|
d = month.end_time.date()
|
|
out = []
|
|
while len(out) < 4:
|
|
if d.weekday() < 5:
|
|
out.append(d)
|
|
d -= timedelta(days=1)
|
|
return out
|
|
|
|
|
|
def _as_of(text: str) -> str | None:
|
|
m = re.search(r'Fund Holdings as of,"([^"]+)"', text)
|
|
if not m or m.group(1).strip() == "-":
|
|
return None
|
|
return datetime.strptime(m.group(1), "%b %d, %Y").strftime("%Y%m%d")
|
|
|
|
|
|
def fetch_holdings() -> None:
|
|
os.makedirs(HOLD_DIR, exist_ok=True)
|
|
idx_path = os.path.join(HOLD_DIR, "index.json")
|
|
index = json.load(open(idx_path)) if os.path.exists(idx_path) else {}
|
|
today = date.today()
|
|
for fund, pid in FUNDS.items():
|
|
fdir = os.path.join(HOLD_DIR, fund)
|
|
os.makedirs(fdir, exist_ok=True)
|
|
seen = index.setdefault(fund, {})
|
|
months = pd.period_range(FIRST_MONTH, pd.Period(today, "M") - 1, freq="M")
|
|
# recent dates the product page lists (the latest may be mid-month)
|
|
meta = json.loads(_get(DATES_API.format(pid=pid), {"Accept": "application/json",
|
|
"x-application-id": "pp-ui-csr"}))
|
|
listed = meta["componentsByNameMap"]["holdings"]["containersByNameMap"]["all"][
|
|
"dataPointsByNameMap"]["dateList"]["value"]
|
|
time.sleep(1.2)
|
|
todo = [(str(m), _month_end_candidates(m)) for m in months]
|
|
todo += [(f"listed-{d}", [datetime.strptime(str(d), "%Y%m%d").date()]) for d in listed]
|
|
for key, cands in todo:
|
|
if key in seen:
|
|
continue
|
|
got = None
|
|
for d in cands:
|
|
if d >= today:
|
|
continue
|
|
ds = d.strftime("%Y%m%d")
|
|
path = os.path.join(fdir, f"{ds}.csv")
|
|
if os.path.exists(path):
|
|
got = ds
|
|
break
|
|
text = _get(API.format(pid=pid, d=ds)).decode("utf-8-sig", "replace")
|
|
time.sleep(1.2)
|
|
asof = _as_of(text)
|
|
if asof is None:
|
|
continue
|
|
if asof != ds:
|
|
print(f" {fund} {ds}: file says {asof}", flush=True)
|
|
with open(os.path.join(fdir, f"{asof}.csv"), "w") as fh:
|
|
fh.write(text)
|
|
got = asof
|
|
break
|
|
seen[key] = got
|
|
print(f"{fund} {key}: {got or 'none'}", flush=True)
|
|
json.dump(index, open(idx_path, "w"), indent=1, sort_keys=True)
|
|
json.dump(index, open(idx_path, "w"), indent=1, sort_keys=True)
|
|
parse_all()
|
|
|
|
|
|
def _num(x: str) -> float:
|
|
x = (x or "").replace(",", "").strip()
|
|
try:
|
|
return float(x)
|
|
except ValueError:
|
|
return np.nan
|
|
|
|
|
|
def parse_snapshot(path: str) -> pd.DataFrame:
|
|
"""All rows of one holdings CSV, numbers parsed."""
|
|
rows = list(csv.reader(io.open(path, encoding="utf-8-sig")))
|
|
h = next(i for i, r in enumerate(rows) if r and r[0] == "Ticker")
|
|
shares_out = next((_num(r[1]) for r in rows[:h] if len(r) > 1 and r[0] == "Shares Outstanding"),
|
|
np.nan)
|
|
head = rows[h]
|
|
body = [r for r in rows[h + 1:] if len(r) == len(head)]
|
|
df = pd.DataFrame(body, columns=head)
|
|
wcol = next((c for c in ("Weight (%)", "Market Weight") if c in df.columns), None)
|
|
return pd.DataFrame({
|
|
"shares_out": shares_out,
|
|
"ticker_raw": df["Ticker"].str.strip(),
|
|
"name": df["Name"].str.strip().str.upper(),
|
|
"sector": df["Sector"].str.strip(),
|
|
"asset_class": df["Asset Class"].str.strip(),
|
|
"type": df["Type"].str.strip() if "Type" in df else "-",
|
|
"quantity": df["Quantity"].map(_num) if "Quantity" in df else np.nan,
|
|
"price": df["Price"].map(_num) if "Price" in df else np.nan,
|
|
"market_value": df["Market Value"].map(_num) if "Market Value" in df else np.nan,
|
|
"weight_reported": df[wcol].map(_num) if wcol else np.nan,
|
|
})
|
|
|
|
|
|
def _is_equity(df: pd.DataFrame) -> pd.Series:
|
|
"""Stocks only: drops cash, money-market funds, futures, swaps and
|
|
collateral. A few old rows have no asset class (and no ticker) but a GICS
|
|
sector and a share price; they are stocks."""
|
|
typ_ok = df["type"].isin(["EQUITY", "-", ""])
|
|
listed = df["asset_class"] == "Equity"
|
|
unclassed = (df["asset_class"].isin(["-", ""]) & ~df["sector"].isin(CASH_SECTORS | {"-", ""})
|
|
& (df["price"] > 1.0) & (df["price"] != 100.0))
|
|
not_stock = df["name"].str.contains(NOT_STOCK, regex=True, na=False)
|
|
return typ_ok & (listed | unclassed) & ~not_stock
|
|
|
|
|
|
_CLEAN = re.compile(r"[A-Z]{1,5}([ .][A-Z])?")
|
|
|
|
|
|
def _yahoo_symbol(raw: str) -> str | None:
|
|
"""BlackRock ticker → Yahoo symbol. Class shares use a dash (BRK.B →
|
|
BRK-B, "MOG A" → MOG-A); BlackRock's disambiguation suffixes are dropped
|
|
(VRX.1 → VRX, PSYS. → PSYS, LKQ1 → LKQ, "CC WI" → CC). For suffixed
|
|
codes this is the last resort: ``_resolve`` first looks the company name
|
|
up (WM1 is Williams-Sonoma, WSM)."""
|
|
t = re.sub(r"\s+", " ", raw.strip().upper())
|
|
if t in ("", "-") or re.search(r"[ ./](WS|RT|RTS)$", t): # rights and warrants
|
|
return None
|
|
t = re.sub(r"[ .](WI|US)$", "", t) # when-issued lines, "EHTH US"
|
|
m = re.fullmatch(r"([A-Z]+)[ .]([A-Z])", t)
|
|
if m:
|
|
return f"{m.group(1)}-{m.group(2)}"
|
|
t = re.sub(r"\.\d*$", "", t) # VRX.1, PSYS.
|
|
m = re.fullmatch(r"([A-Z]+)\d+([A-Z]?)", t) # LKQ1, RS6, FR1A
|
|
if m:
|
|
t = m.group(1) + (f"-{m.group(2)}" if m.group(2) else "")
|
|
t = t.replace(".", "-").replace(" ", "-").replace("/", "-")
|
|
return t or None
|
|
|
|
|
|
def _norm_name(n: str) -> str:
|
|
n = str(n).upper().replace("&", " AND ")
|
|
n = re.sub(r"\bCL(ASS)?\s+([A-Z])\b", r"CL\2", n)
|
|
n = re.sub(r"[^A-Z0-9 ]", " ", n)
|
|
n = re.sub(r"\b(INC|INCORPORATED|CORP|CORPORATION|CO|COMPANY|LTD|LIMITED|PLC|THE|NEW|"
|
|
r"HOLDINGS?|HLDGS?|GROUP|GRP|INTL|INTERNATIONAL)\b", " ", n)
|
|
return " ".join(n.split())
|
|
|
|
|
|
def _resolve(rows: pd.DataFrame) -> pd.Series:
|
|
"""Yahoo symbol per row. Rows whose ticker is a clean symbol keep it;
|
|
suffixed codes and missing tickers take the clean symbol the same
|
|
company name carries in any snapshot (exact normalized name, else a
|
|
unique prefix match of at least 10 characters), else the stripped code."""
|
|
raw = rows["ticker_raw"].fillna("").str.strip().str.upper()
|
|
clean = raw.str.fullmatch(_CLEAN)
|
|
nm = rows["name"].map(_norm_name)
|
|
sym = raw.map(_yahoo_symbol)
|
|
by_name = (pd.DataFrame({"nm": nm[clean], "sym": sym[clean]})
|
|
.groupby("nm")["sym"].agg(lambda s: s.mode().iloc[0]))
|
|
names = sorted(by_name.index)
|
|
out = sym.where(clean)
|
|
for n in nm[~clean].unique():
|
|
hit = by_name.get(n)
|
|
if hit is None and len(n) >= 10:
|
|
cand = {by_name[k] for k in names if k.startswith(n) or (len(k) >= 10 and n.startswith(k))}
|
|
hit = cand.pop() if len(cand) == 1 else None
|
|
if hit is not None:
|
|
out[(~clean) & (nm == n)] = hit
|
|
left = out.isna()
|
|
out[left] = sym[left]
|
|
return out
|
|
|
|
|
|
def parse_all() -> pd.DataFrame:
|
|
parts = []
|
|
for fund in FUNDS:
|
|
fdir = os.path.join(HOLD_DIR, fund)
|
|
for f in sorted(os.listdir(fdir)) if os.path.isdir(fdir) else []:
|
|
if f.endswith(".csv"):
|
|
df = parse_snapshot(os.path.join(fdir, f))
|
|
df.insert(0, "date", pd.Timestamp(f[:-4]))
|
|
df.insert(0, "fund", fund)
|
|
parts.append(df)
|
|
allrows = pd.concat(parts, ignore_index=True)
|
|
eq = allrows[_is_equity(allrows)].copy()
|
|
eq["mv"] = eq["quantity"] * eq["price"]
|
|
eq["mv"] = eq["market_value"].where(eq["market_value"] > 0, eq["mv"])
|
|
eq["weight"] = eq["mv"] / eq.groupby(["fund", "date"])["mv"].transform("sum")
|
|
eq["ticker"] = _resolve(eq)
|
|
unresolved = eq[eq["ticker"].isna()]
|
|
if len(unresolved):
|
|
print(f"{len(unresolved)} equity rows without a ticker (dropped):",
|
|
sorted(unresolved["name"].unique())[:20])
|
|
eq = eq.dropna(subset=["ticker"]).sort_values("mv", ascending=False)
|
|
# one row per ticker and snapshot (a security held in two lots is summed;
|
|
# the price is the largest lot's)
|
|
eq = (eq.groupby(["fund", "date", "ticker"], as_index=False)
|
|
.agg(name=("name", "first"), ticker_raw=("ticker_raw", "first"),
|
|
sector=("sector", "first"), quantity=("quantity", "sum"), price=("price", "first"),
|
|
mv=("mv", "sum"), weight=("weight", "sum"),
|
|
weight_reported=("weight_reported", "sum"), shares_out=("shares_out", "first")))
|
|
# stub positions: a few hundred shares left after a deletion or a
|
|
# corporate action, not index members (crashed members stay above 2%)
|
|
rel = eq["weight"] * eq.groupby(["fund", "date"])["ticker"].transform("size")
|
|
stub = rel < STUB_REL
|
|
print(f"dropped {stub.sum()} stub positions (< {STUB_REL:.0%} of an equal weight)")
|
|
eq = eq[~stub].copy()
|
|
eq["weight"] = eq["weight"] / eq.groupby(["fund", "date"])["weight"].transform("sum")
|
|
eq.drop(columns="shares_out").to_csv(HOLDINGS, index=False)
|
|
summ = eq.groupby(["fund", "date"]).agg(n=("ticker", "size"), equity_mv=("mv", "sum"),
|
|
weight_reported=("weight_reported", "sum"),
|
|
shares_out=("shares_out", "first"))
|
|
summ.to_csv(os.path.join(HOLD_DIR, "snapshots.csv"))
|
|
print(f"Saved {HOLDINGS}: {len(eq)} rows, {eq['ticker'].nunique()} tickers")
|
|
return eq
|
|
|
|
|
|
def load_holdings() -> pd.DataFrame:
|
|
return pd.read_csv(HOLDINGS, parse_dates=["date"])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# prices
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _px_path(t: str) -> str:
|
|
return os.path.join(PX_DIR, t.replace("^", "_").replace("=", "_").replace("/", "_") + ".csv")
|
|
|
|
|
|
def _no_market_proxy() -> None:
|
|
"""Never fetch Yahoo through QUANT_MARKET_PROXY (the live monitor's IP)."""
|
|
mp = os.environ.get("QUANT_MARKET_PROXY", "")
|
|
if not mp and os.path.exists(".env"):
|
|
for line in open(".env"):
|
|
if line.startswith("QUANT_MARKET_PROXY="):
|
|
mp = line.split("=", 1)[1].strip().strip("'\"")
|
|
for k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"):
|
|
if mp and os.environ.get(k, "").rstrip("/") == mp.rstrip("/"):
|
|
del os.environ[k]
|
|
|
|
|
|
def fetch_prices(start: str = "2005-01-01", retry_missing: bool = False) -> None:
|
|
import yfinance as yf
|
|
from yfinance.exceptions import YFPricesMissingError, YFTzMissingError
|
|
yf.config.debug.hide_exceptions = False
|
|
_no_market_proxy()
|
|
os.makedirs(PX_DIR, exist_ok=True)
|
|
miss_path = os.path.join(PX_DIR, "missing.json")
|
|
missing = json.load(open(miss_path)) if os.path.exists(miss_path) else {}
|
|
tickers = ETFS + sorted(set(load_holdings()["ticker"]) - set(ETFS))
|
|
todo = [t for t in tickers if not os.path.exists(_px_path(t))
|
|
and (retry_missing or t not in missing)]
|
|
print(f"{len(tickers)} tickers, {len(todo)} to fetch", flush=True)
|
|
errors = 0
|
|
miss_streak: list[str] = []
|
|
i = 0
|
|
while i < len(todo):
|
|
t = todo[i]
|
|
try:
|
|
h = yf.Ticker(t).history(start=start, auto_adjust=False, actions=True)
|
|
if len(h) == 0:
|
|
raise YFPricesMissingError(t, "empty")
|
|
h.index = pd.DatetimeIndex(h.index.tz_localize(None).normalize(), name="Date")
|
|
h.to_csv(_px_path(t))
|
|
missing.pop(t, None)
|
|
print(f"[{i + 1}/{len(todo)}] {t:8s} ok {len(h)} bars {h.index[0].date()}", flush=True)
|
|
errors, miss_streak = 0, []
|
|
except Exception as exc: # noqa: BLE001 — sorted below
|
|
msg = f"{type(exc).__name__}: {exc}"
|
|
gone = (isinstance(exc, (YFTzMissingError, YFPricesMissingError))
|
|
or "404" in msg or "Not Found" in msg or "delisted" in msg)
|
|
if not gone: # rate limit or network error: wait, retry the same ticker
|
|
errors += 1
|
|
print(f"[{i + 1}/{len(todo)}] {t:8s} error {msg[:140]}", flush=True)
|
|
json.dump(missing, open(miss_path, "w"), indent=0, sort_keys=True)
|
|
time.sleep(600 if errors >= 5 else 60)
|
|
if errors >= 5:
|
|
errors = 0
|
|
continue
|
|
missing[t] = datetime.now().isoformat(timespec="seconds")
|
|
miss_streak.append(t)
|
|
print(f"[{i + 1}/{len(todo)}] {t:8s} missing", flush=True)
|
|
if len(miss_streak) >= 25: # a long run of misses: is Yahoo throttling?
|
|
try:
|
|
ok = len(yf.Ticker("SPY").history(period="5d")) > 0
|
|
except Exception: # noqa: BLE001
|
|
ok = False
|
|
if not ok:
|
|
print(" canary SPY failed: throttled; redoing the streak after 10 min", flush=True)
|
|
for m in miss_streak:
|
|
missing.pop(m, None)
|
|
i -= len(miss_streak) - 1
|
|
miss_streak = []
|
|
time.sleep(600)
|
|
continue
|
|
miss_streak = []
|
|
errors = 0
|
|
if (i + 1) % 25 == 0:
|
|
json.dump(missing, open(miss_path, "w"), indent=0, sort_keys=True)
|
|
i += 1
|
|
time.sleep(1.5)
|
|
json.dump(missing, open(miss_path, "w"), indent=0, sort_keys=True)
|
|
print("done", flush=True)
|
|
|
|
|
|
def load_bars(t: str) -> pd.DataFrame | None:
|
|
"""Adjusted OHLCV (as yfinance's auto_adjust=True) plus the unadjusted close."""
|
|
path = _px_path(t)
|
|
if not os.path.exists(path):
|
|
return None
|
|
h = pd.read_csv(path, index_col=0, parse_dates=True)
|
|
h = h[~h.index.duplicated(keep="last")].sort_index()
|
|
ratio = (h["Adj Close"] / h["Close"]).where(h["Close"] > 0)
|
|
out = pd.DataFrame({k.lower(): h[k] * ratio for k in ("Open", "High", "Low", "Close")})
|
|
out["volume"] = h["Volume"]
|
|
# Yahoo's Close is split-adjusted: undo later splits for the as-traded price
|
|
splits = h["Stock Splits"].replace(0, 1.0).fillna(1.0)
|
|
later = splits[::-1].cumprod()[::-1].shift(-1).fillna(1.0)
|
|
out["raw_close"] = h["Close"] * later
|
|
return out
|
|
|
|
|
|
def status() -> None:
|
|
eq = load_holdings()
|
|
tick = set(eq["ticker"])
|
|
have = {t for t in tick if os.path.exists(_px_path(t))}
|
|
miss_path = os.path.join(PX_DIR, "missing.json")
|
|
missing = set(json.load(open(miss_path))) if os.path.exists(miss_path) else set()
|
|
print(f"tickers {len(tick)}: prices {len(have)}, missing {len(missing & tick)}, "
|
|
f"todo {len(tick - have - missing)}")
|
|
print(snapshot_summary().groupby("fund")
|
|
.agg(first=("date", "min"), last=("date", "max"), snaps=("date", "size"),
|
|
n_min=("n", "min"), n_max=("n", "max"),
|
|
wsum_min=("weight_sum", "min"), wsum_med=("weight_sum", "median"),
|
|
wsum_max=("weight_sum", "max")))
|
|
|
|
|
|
def snapshot_summary() -> pd.DataFrame:
|
|
"""Per snapshot: equity holdings count and their weight sum, the equity
|
|
market value over the fund's assets (the files' own cash rows are
|
|
unusable: cash is sometimes priced at 100). Assets = shares outstanding
|
|
x the ETF's close, both on today's split basis (the header's share count
|
|
is split-adjusted, like Yahoo's unadjusted-for-dividends Close)."""
|
|
s = pd.read_csv(os.path.join(HOLD_DIR, "snapshots.csv"), parse_dates=["date"])
|
|
s["weight_sum"] = np.nan
|
|
for fund in FUNDS:
|
|
if not os.path.exists(_px_path(fund)):
|
|
continue
|
|
close = pd.read_csv(_px_path(fund), index_col=0, parse_dates=True)["Close"]
|
|
m = s["fund"] == fund
|
|
px = close.reindex(close.index.union(s.loc[m, "date"])).ffill().reindex(s.loc[m, "date"])
|
|
s.loc[m, "weight_sum"] = s.loc[m, "equity_mv"].values / (s.loc[m, "shares_out"].values * px.values)
|
|
return s
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("step", choices=["holdings", "parse", "prices", "status"])
|
|
ap.add_argument("--retry-missing", action="store_true")
|
|
args = ap.parse_args()
|
|
if args.step == "holdings":
|
|
fetch_holdings()
|
|
elif args.step == "parse":
|
|
parse_all()
|
|
elif args.step == "prices":
|
|
fetch_prices(retry_missing=args.retry_missing)
|
|
else:
|
|
status()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|