feat: US 3-sleeve portfolio and lowrisk_top20; US portfolio research harness
Register two live strategies from the 2026-09-29 US research: - lowrisk_top20: idiosyncratic vol + low beta, top 20, monthly, buffer 2 (new `low_risk` signal in strategies/robust_momentum.py) - us_three_sleeve: 1/3 each of overnight_recovery_top10, lowrisk_top20 and trend_rider_v7_vt24 in one account (strategies/sleeve_portfolio.py, a mixed stock+ETF strategy). Sleeve shares drift with their returns and reset at a month start once any is >25% off its share. Research (research/lab, S&P 500 PIT, next-open fills): - rotation.py / persistence.py: walk-forward factor weighting (factor momentum, rolling IC, market state) and last-year's-winner selection do not beat a fixed composite - us_sweep.py: holdings count, z/vol weights, dynamic N, tranching and market overlays (Faber SMA, de-risk vol target, statistical jump model, momentum state); engine.py gains zvol weighting and dyn_n - fetch_us_fundamentals.py / fundamentals.py / fund_eval.py: SEC EDGAR point-in-time fundamentals (first-filed values) and factor tests - trend_rider_long.py / us_final.py: 2010-2026 TrendRider curves and the 3-sleeve check (2011-2026 Sharpe 1.21, CAGR 21.9%, MaxDD -23%) Web-UI engine, 2017-09..2026-09, $2/order: us_three_sleeve CAGR 28.9%, Sharpe 1.49, MaxDD -24.8%. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
292
research/lab/fundamentals.py
Normal file
292
research/lab/fundamentals.py
Normal file
@@ -0,0 +1,292 @@
|
||||
"""
|
||||
Point-in-time fundamental factors for the US panel from SEC EDGAR facts
|
||||
(research/lab/fetch_us_fundamentals.py).
|
||||
|
||||
A value is usable from the trading day AFTER its first filing date, so a
|
||||
factor on date t only uses statements filed before t. Durations become
|
||||
quarterly values (direct 3-month facts, or differences of year-to-date facts
|
||||
that share a fiscal-year start) and trailing-twelve-month sums; instants are
|
||||
taken as of their period end.
|
||||
|
||||
from research.lab import fundamentals as fu
|
||||
F = fu.load(panel) # {name: dates x tickers}, cached
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pickle
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
RAW = os.path.join("data", "research", "us_fund_raw.pkl")
|
||||
CACHE = os.path.join("data", "research", "us_fund_factors.pkl")
|
||||
|
||||
DURATION = {
|
||||
"revenue": ["Revenues", "RevenueFromContractWithCustomerExcludingAssessedTax",
|
||||
"RevenueFromContractWithCustomerIncludingAssessedTax", "SalesRevenueNet",
|
||||
"RevenuesNetOfInterestExpense", "SalesRevenueGoodsNet", "SalesRevenueServicesNet"],
|
||||
"cogs": ["CostOfRevenue", "CostOfGoodsAndServicesSold", "CostOfGoodsSold", "CostOfServices"],
|
||||
"gross_profit": ["GrossProfit"],
|
||||
"op_income": ["OperatingIncomeLoss"],
|
||||
"net_income": ["NetIncomeLoss", "ProfitLoss", "NetIncomeLossAvailableToCommonStockholdersBasic"],
|
||||
"cfo": ["NetCashProvidedByUsedInOperatingActivities",
|
||||
"NetCashProvidedByUsedInOperatingActivitiesContinuingOperations"],
|
||||
"capex": ["PaymentsToAcquirePropertyPlantAndEquipment"],
|
||||
"rnd": ["ResearchAndDevelopmentExpense"],
|
||||
"buyback": ["PaymentsForRepurchaseOfCommonStock"],
|
||||
"dividends": ["PaymentsOfDividendsCommonStock", "PaymentsOfDividends"],
|
||||
}
|
||||
INSTANT = {
|
||||
"assets": ["Assets"],
|
||||
"assets_cur": ["AssetsCurrent"],
|
||||
"liab": ["Liabilities"],
|
||||
"liab_cur": ["LiabilitiesCurrent"],
|
||||
"equity": ["StockholdersEquity",
|
||||
"StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest"],
|
||||
"ltd": ["LongTermDebtNoncurrent", "LongTermDebt"],
|
||||
"shares": ["EntityCommonStockSharesOutstanding", "CommonStockSharesOutstanding"],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fact tables → per-ticker event series
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _pick(facts: pd.DataFrame, tags: list[str]) -> pd.DataFrame:
|
||||
"""One value per (ticker, start, end): the highest-priority tag present."""
|
||||
sub = facts[facts["tag"].isin(tags)].copy()
|
||||
sub["prio"] = sub["tag"].map({t: i for i, t in enumerate(tags)})
|
||||
sub = sub.sort_values(["prio", "filed"])
|
||||
return sub.drop_duplicates(["ticker", "start", "end"], keep="first").drop(columns="prio")
|
||||
|
||||
|
||||
def _quarters(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Quarterly values from duration facts of one ticker: direct ~3-month
|
||||
facts, else differences of year-to-date facts sharing a start date."""
|
||||
df = df.dropna(subset=["start"])
|
||||
days = (df["end"] - df["start"]).dt.days
|
||||
direct = df[(days >= 75) & (days <= 105)][["end", "val", "filed"]]
|
||||
derived = []
|
||||
for _, g in df[days > 105].groupby("start"):
|
||||
g = g.sort_values("end")
|
||||
# pair each YTD fact with the one ending ~3 months earlier (same start)
|
||||
allg = pd.concat([g, df[(df["start"] == g["start"].iloc[0]) & (days.loc[df.index] <= 105)]])
|
||||
allg = allg.sort_values("end").drop_duplicates("end")
|
||||
e = allg["end"].values
|
||||
v = allg["val"].values
|
||||
f = allg["filed"].values
|
||||
for k in range(1, len(allg)):
|
||||
gap = (e[k] - e[k - 1]).astype("timedelta64[D]").astype(int)
|
||||
if 75 <= gap <= 105:
|
||||
derived.append((e[k], v[k] - v[k - 1], max(f[k], f[k - 1])))
|
||||
der = pd.DataFrame(derived, columns=["end", "val", "filed"])
|
||||
q = pd.concat([direct.assign(src=0), der.assign(src=1)]).sort_values(["end", "src", "filed"])
|
||||
q = q.drop_duplicates("end", keep="first").drop(columns="src").sort_values("end")
|
||||
# the first quarter of a fiscal year never has a same-start predecessor;
|
||||
# its direct fact covers it
|
||||
return q.reset_index(drop=True)
|
||||
|
||||
|
||||
def _ttm(q: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Sum of four consecutive quarters (ends 75-105 days apart)."""
|
||||
if len(q) < 4:
|
||||
return q.iloc[:0]
|
||||
e = q["end"].values
|
||||
gaps = np.diff(e).astype("timedelta64[D]").astype(int)
|
||||
ok_gap = np.r_[False, (gaps >= 75) & (gaps <= 105)]
|
||||
run = pd.Series(ok_gap).rolling(3).sum().values == 3 # 3 good gaps → 4 quarters
|
||||
vals = q["val"].rolling(4).sum().values
|
||||
filed = pd.to_datetime(q["filed"].astype("int64").rolling(4).max().values)
|
||||
out = pd.DataFrame({"end": e, "val": vals, "filed": filed})
|
||||
return out[run].reset_index(drop=True)
|
||||
|
||||
|
||||
def _asof_panel(events: dict[str, pd.DataFrame], dates: pd.DatetimeIndex,
|
||||
tickers: list[str], lag: int = 0) -> pd.DataFrame:
|
||||
"""Daily panel of the value with the latest period end among facts filed
|
||||
strictly before each date (plus ``lag`` extra periods back: lag=4 gives
|
||||
the value four quarters earlier than the latest)."""
|
||||
out = pd.DataFrame(np.nan, index=dates, columns=tickers)
|
||||
for t, ev in events.items():
|
||||
if t not in out.columns or ev.empty:
|
||||
continue
|
||||
ev = ev.sort_values("filed")
|
||||
# as of each filing: the known history sorted by end
|
||||
rows = []
|
||||
known: dict = {}
|
||||
for end, val, filed in zip(ev["end"].values, ev["val"].values, ev["filed"].values):
|
||||
known[end] = val
|
||||
ends = sorted(known)
|
||||
if len(ends) > lag:
|
||||
rows.append((filed, known[ends[-1 - lag]]))
|
||||
if not rows:
|
||||
continue
|
||||
s = pd.DataFrame(rows, columns=["filed", "val"]).drop_duplicates("filed", keep="last")
|
||||
s = s.set_index("filed")["val"]
|
||||
# usable from the next session after the filing date
|
||||
pos = dates.searchsorted(s.index, side="right")
|
||||
ser = pd.Series(np.nan, index=dates)
|
||||
valid = pos < len(dates)
|
||||
ser.iloc[pos[valid]] = s.values[valid]
|
||||
out[t] = ser.ffill().values
|
||||
return out
|
||||
|
||||
|
||||
def _series_by_ticker(df: pd.DataFrame) -> dict[str, pd.DataFrame]:
|
||||
return {t: g for t, g in df.groupby("ticker")}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def build(p) -> dict[str, pd.DataFrame]:
|
||||
with open(RAW, "rb") as fh:
|
||||
raw = pickle.load(fh)
|
||||
facts = raw["facts"]
|
||||
dates, tickers = p.dates, list(p.close.columns)
|
||||
t0 = time.time()
|
||||
Q, TTM, INS = {}, {}, {}
|
||||
for c, tags in DURATION.items():
|
||||
picked = _pick(facts, tags)
|
||||
q_ev, ttm_ev = {}, {}
|
||||
for t, g in picked.groupby("ticker"):
|
||||
q = _quarters(g)
|
||||
q_ev[t] = q
|
||||
ttm = _ttm(q)
|
||||
# fiscal-year facts give TTM at year ends directly
|
||||
days = (g["end"] - g["start"]).dt.days
|
||||
fy = g[(days >= 350) & (days <= 380)][["end", "val", "filed"]]
|
||||
ttm_ev[t] = (pd.concat([fy.assign(src=0), ttm.assign(src=1)])
|
||||
.sort_values(["end", "src"]).drop_duplicates("end").drop(columns="src"))
|
||||
Q[c], TTM[c] = q_ev, ttm_ev
|
||||
print(f" {c}: {len(q_ev)} tickers [{time.time() - t0:.0f}s]", flush=True)
|
||||
for c, tags in INSTANT.items():
|
||||
if c == "shares":
|
||||
# cover-page and balance-sheet share counts have different dates;
|
||||
# mixing them would break "four reports back", so use one tag per ticker
|
||||
sub = facts[facts["tag"].isin(tags)]
|
||||
INS[c] = {}
|
||||
for t, g in sub.groupby("ticker"):
|
||||
dei = g[g["tag"] == tags[0]]
|
||||
use = dei if len(dei) >= 8 else g[g["tag"] == tags[1]]
|
||||
INS[c][t] = use.drop_duplicates("end")[["end", "val", "filed"]]
|
||||
continue
|
||||
picked = _pick(facts, tags)
|
||||
INS[c] = {t: g[["end", "val", "filed"]] for t, g in picked.groupby("ticker")}
|
||||
|
||||
def ttm(c, lag=0):
|
||||
return _asof_panel(TTM[c], dates, tickers, lag)
|
||||
|
||||
def qv(c, lag=0):
|
||||
return _asof_panel(Q[c], dates, tickers, lag)
|
||||
|
||||
def ins(c, lag=0):
|
||||
return _asof_panel(INS[c], dates, tickers, lag)
|
||||
|
||||
F: dict[str, pd.DataFrame] = {}
|
||||
rev, ni, cfo = ttm("revenue"), ttm("net_income"), ttm("cfo")
|
||||
gp = ttm("gross_profit")
|
||||
cogs = ttm("cogs")
|
||||
gp = gp.combine_first(rev - cogs)
|
||||
opi = ttm("op_income")
|
||||
capex = ttm("capex")
|
||||
assets, equity, liab = ins("assets"), ins("equity"), ins("liab")
|
||||
assets_4 = ins("assets", 4)
|
||||
print(f" panels [{time.time() - t0:.0f}s]", flush=True)
|
||||
pos = lambda x: x.where(x > 0) # noqa: E731
|
||||
|
||||
F["gp_a"] = gp / pos(assets) # Novy-Marx 2013
|
||||
F["op_a"] = opi / pos(assets)
|
||||
F["cfo_a"] = cfo / pos(assets) # cash profitability
|
||||
F["roe"] = ni / pos(equity)
|
||||
F["roa"] = ni / pos(assets)
|
||||
F["accruals"] = -(ni - cfo) / pos(assets) # Sloan 1996 (low is good)
|
||||
F["asset_growth"] = -(assets / pos(assets_4) - 1) # Cooper-Gulen-Schill 2008
|
||||
F["leverage"] = -(liab / pos(assets))
|
||||
F["gross_margin"] = gp / pos(rev)
|
||||
rev_4 = ttm("revenue", 4)
|
||||
F["rev_growth"] = rev / pos(rev_4) - 1
|
||||
gm_4 = ttm("gross_profit", 4).combine_first(rev_4 - ttm("cogs", 4)) / pos(rev_4)
|
||||
F["gm_change"] = F["gross_margin"] - gm_4
|
||||
ni_4 = ttm("net_income", 4)
|
||||
F["roa_change"] = F["roa"] - ni_4 / pos(assets_4)
|
||||
|
||||
# SUE: latest quarterly change vs same quarter last year, scaled by the
|
||||
# std of the last 8 such changes (seasonal random walk, Bernard-Thomas)
|
||||
for c, name in (("net_income", "sue"), ("revenue", "sue_rev")):
|
||||
chg = {}
|
||||
for t, q in Q[c].items():
|
||||
q = q.sort_values("end").reset_index(drop=True)
|
||||
if len(q) < 8:
|
||||
continue
|
||||
e = q["end"].values
|
||||
v = q["val"].values
|
||||
d = np.full(len(q), np.nan)
|
||||
for k in range(4, len(q)):
|
||||
gap = (e[k] - e[k - 4]).astype("timedelta64[D]").astype(int)
|
||||
if 350 <= gap <= 380:
|
||||
d[k] = v[k] - v[k - 4]
|
||||
ds = pd.Series(d)
|
||||
sd = ds.rolling(8, min_periods=4).std()
|
||||
sue = (ds / sd.replace(0, np.nan)).values
|
||||
chg[t] = pd.DataFrame({"end": e, "val": sue, "filed": q["filed"].values}).dropna()
|
||||
F[name] = _asof_panel(chg, dates, tickers).clip(-10, 10)
|
||||
|
||||
# market cap. Yahoo's split-adjusted close is raw / F(t), where F(t) is the
|
||||
# product of split ratios after t, and shares reported at c grow by the
|
||||
# splits in (c, t], so cap_t = shares_c * F(c) * close_sa_t.
|
||||
splits = raw["splits"].reindex(columns=tickers).fillna(0.0)
|
||||
close_sa = raw["close_sa"].reindex(index=dates, columns=tickers).ffill()
|
||||
ratio = splits.replace(0.0, 1.0)
|
||||
# F(c) = product of split ratios strictly after c
|
||||
after = ratio.iloc[::-1].cumprod().iloc[::-1].shift(-1).fillna(1.0)
|
||||
after = after.reindex(dates, method="ffill").fillna(1.0)
|
||||
sh_ev = {}
|
||||
for t, g in INS["shares"].items():
|
||||
if t not in after.columns:
|
||||
continue
|
||||
ff = after[t].reindex(g["end"].values, method="ffill").fillna(1.0).values
|
||||
sh_ev[t] = g.assign(val=g["val"].values * ff) # shares on today's split basis
|
||||
shares_now = _asof_panel(sh_ev, dates, tickers)
|
||||
mcap = shares_now * close_sa
|
||||
mcap = mcap.where(mcap > 1e8)
|
||||
F["mcap"] = mcap
|
||||
F["ep"] = ni / mcap
|
||||
F["cfp"] = cfo / mcap
|
||||
F["sp"] = rev / mcap
|
||||
F["bm"] = equity / mcap
|
||||
F["fcfp"] = (cfo - capex.fillna(0)) / mcap
|
||||
payout = ttm("buyback").fillna(0) + ttm("dividends").fillna(0)
|
||||
F["shareholder_yield"] = payout / mcap
|
||||
sh_4 = _asof_panel(sh_ev, dates, tickers, lag=4)
|
||||
F["net_issuance"] = -np.log(shares_now / sh_4).clip(-1, 1)
|
||||
|
||||
# Piotroski F-score (9 binary signals)
|
||||
ltd, ac, lc = ins("ltd"), ins("assets_cur"), ins("liab_cur")
|
||||
ltd_4, ac_4, lc_4 = ins("ltd", 4), ins("assets_cur", 4), ins("liab_cur", 4)
|
||||
sig = [
|
||||
F["roa"] > 0, F["cfo_a"] > 0, F["roa_change"] > 0, cfo > ni,
|
||||
(ltd / pos(assets)) < (ltd_4 / pos(assets_4)),
|
||||
(ac / pos(lc)) > (ac_4 / pos(lc_4)),
|
||||
F["net_issuance"] >= -0.005,
|
||||
F["gm_change"] > 0,
|
||||
(rev / pos(assets)) > (rev_4 / pos(assets_4)),
|
||||
]
|
||||
avail = F["roa"].notna() & F["cfo_a"].notna()
|
||||
F["fscore"] = sum(s.astype(float) for s in sig).where(avail)
|
||||
print(f" factors built [{time.time() - t0:.0f}s]", flush=True)
|
||||
return {k: v.astype("float32") for k, v in F.items()}
|
||||
|
||||
|
||||
def load(p, rebuild: bool = False) -> dict[str, pd.DataFrame]:
|
||||
if os.path.exists(CACHE) and not rebuild:
|
||||
with open(CACHE, "rb") as fh:
|
||||
return pickle.load(fh)
|
||||
F = build(p)
|
||||
with open(CACHE, "wb") as fh:
|
||||
pickle.dump(F, fh, protocol=4)
|
||||
return F
|
||||
Reference in New Issue
Block a user