Gahow Wang
ad569f1cea
Register us_momentum_qqq_trend: half overnight_recovery_top10, half a QQQ trend sleeve (strategies/qqq_trend.py: 200-day switch times a 30% vol target, held as TQQQ + SHY). Mixed stock+ETF, US only (extra tickers QQQ, SHY, TQQQ). Research (research/lab/): alpha against QQQ with a 2006-16 design / 2017-26 holdout split (alpha.py, qqq_scan.py, beat_qqq.py); leverage and drawdown frontier (levered_book.py, us_target50.py); synthetic 3x ETFs from 1990 and TrendRider/QQQ-trend stress tests (fetch_long_history.py, letf_history.py, vol_trend.py); short-horizon sleeves, diversifiers, industry momentum, earnings-announcement returns; new point-in-time data sources: SEC Form 4 insider trades (fetch_insider.py, insider.py), 10-K/10-Q text similarity, 8-K earnings dates and SIC codes (fetch_lazy_prices.py, lazy_prices.py), and an S&P 400/600 universe from iShares holdings (fetch_us_mid.py, panel_mid.py). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
337 lines
15 KiB
Python
337 lines
15 KiB
Python
"""
|
|
"Lazy Prices" factors for the US panel (Cohen, Malloy & Nguyen 2020, JF):
|
|
firms whose 10-K / 10-Q text barely changes from the comparable prior filing
|
|
("non-changers") outperformed "changers" in 1995-2014.
|
|
|
|
Source: data/research/lazy_prices.csv from research/lab/fetch_lazy_prices.py
|
|
(SEC EDGAR primary documents; each 10-K vs the previous 10-K, each 10-Q vs the
|
|
10-Q of the same fiscal quarter a year earlier).
|
|
|
|
Point-in-time rule: a filing's similarity is usable from the first trading
|
|
day strictly after its EDGAR filing date, and the most recent filing's value
|
|
is held until the next filing (a newer filing without a comparable prior one
|
|
resets it to NaN). Values older than 400 days are dropped. Filings whose own
|
|
or reference text has fewer than 500 words (exhibit-only shells, cover pages
|
|
of documents incorporated by reference) get no value.
|
|
|
|
from research.lab import lazy_prices as lp
|
|
F = lp.load(panel) # {name: dates x tickers}, cached
|
|
# lazy_cos cosine similarity of term counts, latest 10-K or 10-Q (higher = fewer changes)
|
|
# lazy_jac Jaccard similarity of word sets, latest 10-K or 10-Q
|
|
# lazy_cos_10k cosine similarity, latest 10-K only
|
|
# lazy_rf cosine similarity of Item 1A Risk Factors, latest 10-K only
|
|
# lazy_cos_pct lazy_cos as a percentile among same-form filings of the past
|
|
# 365 days (10-Qs are less similar than 10-Ks: median cosine
|
|
# 0.97 vs 0.99, so the raw mix partly sorts on form type)
|
|
# lazy_jac_pct the same for lazy_jac (median 0.78 for 10-Qs vs 0.88)
|
|
|
|
Evaluation (coverage, monthly quintile sorts and rank IC, the same sorts on
|
|
within-sector ranks, long-only top 20, correlation with the momentum sleeve)
|
|
for 2006-2014, 2015-2026 and the whole span; writes data/research/results/lazy_prices_eval.json:
|
|
|
|
PYTHONPATH=. uv run python -m research.lab.lazy_prices
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import pickle
|
|
import warnings
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from scipy import stats as sps
|
|
|
|
from research.lab.fetch_lazy_prices import GROUPS, OUT as RAW
|
|
from research.lab.report import stats
|
|
|
|
warnings.filterwarnings("ignore")
|
|
|
|
CACHE = os.path.join("data", "research", "lazy_prices_factors.pkl")
|
|
MAX_AGE = 400
|
|
MIN_WORDS = 500
|
|
FACTORS = { # name: (similarity column, forms)
|
|
"lazy_cos": ("cos_sim", GROUPS["10-K"] + GROUPS["10-Q"]),
|
|
"lazy_jac": ("jac_sim", GROUPS["10-K"] + GROUPS["10-Q"]),
|
|
"lazy_cos_10k": ("cos_sim", GROUPS["10-K"]),
|
|
"lazy_rf": ("cos_rf", GROUPS["10-K"]),
|
|
"lazy_cos_pct": ("cos_pct", GROUPS["10-K"] + GROUPS["10-Q"]),
|
|
"lazy_jac_pct": ("jac_pct", GROUPS["10-K"] + GROUPS["10-Q"]),
|
|
}
|
|
PERIODS = {"2006-2014": ("2006-01-01", "2014-12-31"), "2015-2026": ("2015-01-01", "2026-09-24"),
|
|
"2006-2026": ("2006-01-01", "2026-09-24")}
|
|
BLOCKS = [("2006-01-03", "2010-12-31"), ("2011-01-01", "2014-12-31"), ("2015-01-01", "2018-12-31"),
|
|
("2019-01-01", "2022-12-31"), ("2023-01-01", "2026-09-24")]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# factor panels
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def filings() -> pd.DataFrame:
|
|
"""One row per (ticker, filing) with the word-count screen applied."""
|
|
df = pd.read_csv(RAW, dtype={"period": str, "ref_accession": str})
|
|
df["filed"] = pd.to_datetime(df["filed"])
|
|
words = df.drop_duplicates("accession").set_index("accession")["n_words"]
|
|
ref_words = df["ref_accession"].map(words)
|
|
short = (df["n_words"] < MIN_WORDS) | (ref_words < MIN_WORDS)
|
|
df.loc[short, ["cos_sim", "jac_sim", "cos_rf", "jac_rf"]] = np.nan
|
|
df["cos_pct"] = _trailing_pct(df, "cos_sim")
|
|
df["jac_pct"] = _trailing_pct(df, "jac_sim")
|
|
return df
|
|
|
|
|
|
def _trailing_pct(df: pd.DataFrame, col: str, days: int = 365) -> pd.Series:
|
|
"""A similarity as the share of same-form filings filed in the past
|
|
``days`` days (through the same filing date) with a value at or below it."""
|
|
out = pd.Series(np.nan, index=df.index)
|
|
u = df.drop_duplicates("accession").dropna(subset=[col])
|
|
pct = {}
|
|
for forms in GROUPS.values():
|
|
g = u[u["form"].isin(forms)].sort_values("filed")
|
|
f = g["filed"].values
|
|
v = g[col].values
|
|
lo = np.searchsorted(f, f - np.timedelta64(days, "D"), side="left")
|
|
hi = np.searchsorted(f, f, side="right")
|
|
for acc, a, b, x in zip(g["accession"].values, lo, hi, v):
|
|
pct[acc] = (v[a:b] <= x).mean()
|
|
ok = df[col].notna()
|
|
out[ok] = df.loc[ok, "accession"].map(pct)
|
|
return out
|
|
|
|
|
|
def _asof(ev: pd.DataFrame, col: str, dates: pd.DatetimeIndex, tickers: list[str]) -> pd.DataFrame:
|
|
"""Latest filing's value from the session after its filing date; NaN
|
|
values reset, and values older than MAX_AGE days drop out."""
|
|
out = np.full((len(dates), len(tickers)), np.nan, dtype="float32")
|
|
cols = {t: i for i, t in enumerate(tickers)}
|
|
day = dates.values.astype("datetime64[D]").astype("int64")
|
|
for t, g in ev.groupby("ticker"):
|
|
if t not in cols:
|
|
continue
|
|
g = g.sort_values("filed")
|
|
pos = dates.searchsorted(g["filed"].values, side="right")
|
|
ok = pos < len(dates)
|
|
pos, vals = pos[ok], g[col].values[ok]
|
|
fday = g["filed"].values[ok].astype("datetime64[D]").astype("int64")
|
|
if not len(pos):
|
|
continue
|
|
# several filings before one session: the last one wins
|
|
last = np.r_[pos[1:] != pos[:-1], True]
|
|
pos, vals, fday = pos[last], vals[last], fday[last]
|
|
k = np.full(len(dates), -1)
|
|
k[pos] = np.arange(len(pos))
|
|
k = np.maximum.accumulate(k)
|
|
have = k >= 0
|
|
v = np.full(len(dates), np.nan)
|
|
v[have] = vals[k[have]]
|
|
v[have & (day - np.where(have, fday[np.maximum(k, 0)], 0) > MAX_AGE)] = np.nan
|
|
out[:, cols[t]] = v
|
|
return pd.DataFrame(out, index=dates, columns=tickers)
|
|
|
|
|
|
def build(p) -> dict[str, pd.DataFrame]:
|
|
df = filings()
|
|
dates, tickers = p.dates, list(p.close.columns)
|
|
return {name: _asof(df[df["form"].isin(forms)], col, dates, tickers)
|
|
for name, (col, forms) in FACTORS.items()}
|
|
|
|
|
|
def load(p, rebuild: bool = False) -> dict[str, pd.DataFrame]:
|
|
if os.path.exists(CACHE) and not rebuild:
|
|
with open(CACHE, "rb") as fh:
|
|
return pickle.load(fh)
|
|
F = build(p)
|
|
with open(CACHE, "wb") as fh:
|
|
pickle.dump(F, fh, protocol=4)
|
|
return F
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# evaluation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def nw_t(x, lags: int | None = None) -> float:
|
|
"""Newey-West t-statistic of the mean (Bartlett kernel)."""
|
|
x = np.asarray(pd.Series(x).dropna(), dtype=float)
|
|
T = len(x)
|
|
if T < 12:
|
|
return np.nan
|
|
L = lags if lags is not None else int(4 * (T / 100) ** (2 / 9))
|
|
e = x - x.mean()
|
|
s = e @ e / T
|
|
for lag in range(1, L + 1):
|
|
s += 2 * (1 - lag / (L + 1)) * (e[lag:] @ e[:-lag]) / T
|
|
return float(x.mean() / np.sqrt(s / T))
|
|
|
|
|
|
def monthly_returns(p) -> tuple[pd.DatetimeIndex, pd.DataFrame]:
|
|
"""Signal on each month's last session, filled at the next open, held to
|
|
the open after the next month's last session."""
|
|
o = p.open.where(p.open > 0).ffill()
|
|
me = pd.DatetimeIndex(p.dates.to_series().groupby(p.dates.to_period("M")).max().values)
|
|
pos = p.dates.searchsorted(me, side="right")
|
|
ok = pos < len(p.dates)
|
|
me, pos = me[ok], pos[ok]
|
|
fill = o.iloc[pos].values
|
|
R = pd.DataFrame(fill[1:] / fill[:-1] - 1, index=me[:-1], columns=p.close.columns)
|
|
return me[:-1], R
|
|
|
|
|
|
def quintiles(score: pd.DataFrame, R: pd.DataFrame, member: pd.DataFrame,
|
|
min_names: int = 50) -> pd.DataFrame:
|
|
"""Per month: EW return of each quintile minus the EW member average,
|
|
Q5-Q1, and the rank IC (Q1 = lowest similarity = most changes)."""
|
|
rows = {}
|
|
for d in R.index:
|
|
m = member.loc[d]
|
|
r = R.loc[d].where(m)
|
|
univ = r.mean()
|
|
s = score.loc[d].where(m & r.notna()).dropna()
|
|
if len(s) < min_names:
|
|
continue
|
|
q = np.floor((s.rank(method="first") - 1) / len(s) * 5).astype(int)
|
|
row = {f"Q{k + 1}": r[q.index[q == k]].mean() - univ for k in range(5)}
|
|
row["Q5-Q1"] = row["Q5"] - row["Q1"]
|
|
row["ic"] = sps.spearmanr(s.values, r[s.index].values).statistic
|
|
row["n"] = len(s)
|
|
rows[d] = row
|
|
cols = ["Q1", "Q2", "Q3", "Q4", "Q5", "Q5-Q1", "ic", "n"]
|
|
return pd.DataFrame(rows, index=cols).T.astype(float)
|
|
|
|
|
|
def _qtable(Q: pd.DataFrame) -> dict:
|
|
out = {}
|
|
for c in ["Q1", "Q2", "Q3", "Q4", "Q5", "Q5-Q1"]:
|
|
out[c] = {"mean_bp": float(Q[c].mean() * 1e4), "t_nw": nw_t(Q[c]),
|
|
"t": float(Q[c].mean() / Q[c].std() * np.sqrt(Q[c].count()))}
|
|
out["ic"] = {"mean": float(Q["ic"].mean()), "t_nw": nw_t(Q["ic"]),
|
|
"t": float(Q["ic"].mean() / Q["ic"].std() * np.sqrt(Q["ic"].count())),
|
|
"hit": float((Q["ic"] > 0).mean())}
|
|
out["months"] = int(len(Q))
|
|
out["avg_names"] = float(Q["n"].mean())
|
|
return out
|
|
|
|
|
|
def coverage(F: pd.DataFrame, member: pd.DataFrame) -> dict:
|
|
have = (F.notna() & member).sum(axis=1)
|
|
mem = member.sum(axis=1)
|
|
y = pd.DataFrame({"have": have, "mem": mem}).loc["2005":].groupby(lambda d: d.year).mean()
|
|
return {int(k): {"members": round(float(r["mem"]), 1), "with_value": round(float(r["have"]), 1),
|
|
"share": round(float(r["have"] / r["mem"]), 3)} for k, r in y.iterrows()}
|
|
|
|
|
|
def _monthly(r: pd.Series) -> pd.Series:
|
|
return (1 + r).resample("ME").prod() - 1
|
|
|
|
|
|
def main() -> None:
|
|
from research.lab.factor_scan import RESULTS, load as load_panel
|
|
from research.lab.rotation import factor_ranks
|
|
from research.lab.us_final import describe
|
|
from research.lab.us_sweep import _blend, run, sector_rank
|
|
|
|
p, _ = load_panel("us")
|
|
F = load(p, rebuild=True)
|
|
df = filings()
|
|
out: dict = {"n_filings": {}, "similarity_by_form": {}}
|
|
for g, forms in GROUPS.items():
|
|
sub = df[df["form"].isin(forms)].drop_duplicates("accession")
|
|
out["n_filings"][g] = {"total": int(len(sub)), "with_cos": int(sub["cos_sim"].notna().sum()),
|
|
"with_rf": int(sub["cos_rf"].notna().sum())}
|
|
by_y = sub.groupby(sub["filed"].dt.year)[["cos_sim", "jac_sim", "cos_rf"]].median()
|
|
out["similarity_by_form"][g] = {int(y): {k: round(float(v), 4) for k, v in r.items()}
|
|
for y, r in by_y.iterrows()}
|
|
print("filings:", out["n_filings"])
|
|
|
|
# coverage
|
|
out["coverage"] = {n: coverage(f, p.member) for n, f in F.items()}
|
|
print("\n=== coverage: PIT members with a value (yearly mean of daily counts)")
|
|
print("year members " + " ".join(f"{n:>13s}" for n in F))
|
|
for y in out["coverage"]["lazy_cos"]:
|
|
c = out["coverage"]
|
|
print(f"{y} {c['lazy_cos'][y]['members']:7.0f} "
|
|
+ " ".join(f"{c[n][y]['with_value']:6.0f} ({c[n][y]['share']:4.0%})" for n in F))
|
|
|
|
# quintile sorts and IC
|
|
_, R = monthly_returns(p)
|
|
out["quintiles"] = {}
|
|
print("\n=== monthly quintiles, EW among PIT members: mean excess over member EW, bp/month "
|
|
"[NW t]; Q1 = most changes, Q5 = fewest")
|
|
for n, f in F.items():
|
|
Q = quintiles(f, R, p.member)
|
|
out["quintiles"][n] = {}
|
|
for per, (a, b) in PERIODS.items():
|
|
tb = _qtable(Q.loc[a:b])
|
|
out["quintiles"][n][per] = tb
|
|
print(f"{n:13s} {per}: " + " ".join(
|
|
f"{c} {tb[c]['mean_bp']:+5.1f} [{tb[c]['t_nw']:+.1f}]" for c in ["Q1", "Q2", "Q3", "Q4", "Q5"])
|
|
+ f" | Q5-Q1 {tb['Q5-Q1']['mean_bp']:+5.1f} [{tb['Q5-Q1']['t_nw']:+.2f}]"
|
|
+ f" IC {tb['ic']['mean']:+.4f} [{tb['ic']['t_nw']:+.2f}] n={tb['avg_names']:.0f}")
|
|
yr = Q.groupby(Q.index.year)[["Q5-Q1", "ic"]].mean()
|
|
out["quintiles"][n]["by_year"] = {int(y): {"q5_q1_bp": float(r["Q5-Q1"] * 1e4), "ic": float(r["ic"])}
|
|
for y, r in yr.iterrows()}
|
|
|
|
# robustness: the same sorts on within-sector ranks (GICS, PIT members)
|
|
out["quintiles_sector_neutral"] = {}
|
|
print("\n=== sector-neutral ranks: Q5-Q1 bp/month [NW t], IC [NW t]")
|
|
for n, f in F.items():
|
|
Q = quintiles(sector_rank(f, p), R, p.member)
|
|
out["quintiles_sector_neutral"][n] = {per: _qtable(Q.loc[a:b]) for per, (a, b) in PERIODS.items()}
|
|
print(f"{n:13s} " + " ".join(
|
|
f"{per}: {tb['Q5-Q1']['mean_bp']:+5.1f} [{tb['Q5-Q1']['t_nw']:+.2f}] IC {tb['ic']['mean']:+.4f} "
|
|
f"[{tb['ic']['t_nw']:+.2f}]" for per, tb in out["quintiles_sector_neutral"][n].items()))
|
|
|
|
# long-only top 20 vs the equal-weight universe
|
|
span = [(BLOCKS[0][0], BLOCKS[-1][1])]
|
|
ew = p.close.notna().astype(float).where(p.member)
|
|
runs = {n: run(p, f.where(p.member), BLOCKS, top_n=20, buffer=2.0, every=21) for n, f in F.items()}
|
|
runs["ew_members"] = run(p, ew, BLOCKS, top_n=10000, every=21)
|
|
pr = factor_ranks(p, "us", ["overnight_mom", "rec_126"])
|
|
mom = _blend(pr["overnight_mom"], pr["rec_126"])
|
|
runs["momentum_sleeve"] = run(p, mom.where(p.member), span, top_n=10, buffer=2.0, every=10)
|
|
ew_r = runs["ew_members"]["_rets"]
|
|
out["top20"] = {}
|
|
print("\n=== long-only top 20 (every 21, buffer 2, next-open fills, 10 bp), EW members for reference")
|
|
print(f"{'':16s} " + " ".join(f"{per:^34s}" for per in PERIODS) + " blocks (sharpe)")
|
|
for n, st in runs.items():
|
|
r = st["_rets"]
|
|
row = {"turn": float(st["turn"]),
|
|
"block_sharpe": {f"{a[:4]}-{b[:4]}": float(v) for (a, b), v in zip(BLOCKS, st["blocks"])}
|
|
if n != "momentum_sleeve" else {}}
|
|
cells = []
|
|
for per, (a, b) in PERIODS.items():
|
|
d = describe(r.loc[a:b], blocks=[])
|
|
xs = (r - ew_r).loc[a:b]
|
|
ir = float(xs.mean() / xs.std() * np.sqrt(252)) if xs.std() > 0 else np.nan
|
|
row[per] = {"cagr": d["cagr"], "sharpe": d["sharpe"], "mdd": d["mdd"], "vol": d["vol"],
|
|
"excess_ann": float(xs.mean() * 252), "ir_vs_ew": ir}
|
|
cells.append(f"cagr {d['cagr']*100:5.1f}% sh {d['sharpe']:.2f} IR {ir:+.2f} ")
|
|
out["top20"][n] = row
|
|
bl = " ".join(f"{v:.2f}" for v in row["block_sharpe"].values())
|
|
print(f"{n:16s} " + " ".join(cells) + f" [{bl}] turn {st['turn']:.1f}x")
|
|
|
|
# correlation with the momentum sleeve (monthly)
|
|
M = pd.DataFrame({n: _monthly(st["_rets"]) for n, st in runs.items()})
|
|
X = M.sub(M["ew_members"], axis=0)
|
|
out["corr_with_momentum"] = {}
|
|
print("\n=== monthly correlation with the momentum sleeve (raw / excess over EW members)")
|
|
for n in F:
|
|
row = {}
|
|
for per, (a, b) in PERIODS.items():
|
|
row[per] = {"raw": float(M.loc[a:b, n].corr(M.loc[a:b, "momentum_sleeve"])),
|
|
"excess": float(X.loc[a:b, n].corr(X.loc[a:b, "momentum_sleeve"]))}
|
|
out["corr_with_momentum"][n] = row
|
|
print(f"{n:13s} " + " ".join(f"{per}: {v['raw']:+.2f} / {v['excess']:+.2f}" for per, v in row.items()))
|
|
|
|
os.makedirs(RESULTS, exist_ok=True)
|
|
path = os.path.join(RESULTS, "lazy_prices_eval.json")
|
|
with open(path, "w") as fh:
|
|
json.dump(out, fh, indent=1, default=float)
|
|
print(f"\nsaved {path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|