feat(research): supply-chain links, theme propagation, cycles, attention, small caps
Tests of the "rank and buy top N misses the chain / cycle / sentiment" question (report: https://claude.ai/artifact/HUnPtcy7D7Zp1yNQsDLRHY): - review_live_us: Apr-Sep 2026 replay; book return vs AI-chain weight corr 0.71, overnight_recovery_top10 excess almost all allocation (memory stage +28pp). - fetch_supply_links: 10-K mentions of S&P 1500 companies by role, PIT by filing date (12,117 S&P 500 10-Ks; --universe mid for 14,382 S&P 400/600 10-Ks). - lead_lag: corr-peer and 10-K link momentum worked 2006-16 (FM t 3.3-4.3) and vanished 2017-26 (t -2.1..-0.6); Granger lead-lag network and earnings spillover drift null after 2016. - theme_propagation: linked laggards of each era's hot industry underperform 2017-26 (-2% to -12%/yr). - lead_lag_small: the same signals on S&P 400/600, 2019-26, all |t| <= 1.8. - sector_cycle: 21 cycle/rotation variants, none with holdout |t| > 2. - fetch_wiki_attention + attention: Wikipedia pageview attention, |IC| <= 0.013. - concentration: industry caps on momentum top 10 cut alpha, not drawdown. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
350
research/lab/fetch_supply_links.py
Normal file
350
research/lab/fetch_supply_links.py
Normal file
@@ -0,0 +1,350 @@
|
||||
"""
|
||||
Economic links between S&P 500 ever-members from their 10-K filings: which
|
||||
other member companies each 10-K names, and in what role (customer, supplier,
|
||||
competitor). The links become known on the filing date, so a signal built
|
||||
from them is point-in-time.
|
||||
|
||||
PYTHONPATH=. uv run python -m research.lab.fetch_supply_links --workers 4
|
||||
PYTHONPATH=. uv run python -m research.lab.fetch_supply_links --universe mid # S&P 400/600 10-Ks from 2015
|
||||
|
||||
Inputs: data/research/lazy_prices_index.csv (10-K accessions and primary
|
||||
documents, from research/lab/fetch_lazy_prices.py) and sec_sic.csv (ticker,
|
||||
CIK, current SEC name). Names: current and former SEC names (submissions JSON,
|
||||
cached in data/research/sec_names.json), the Wikipedia security name of
|
||||
current members, and a short manual list of brands (Google, Facebook, AWS).
|
||||
Matching is case-sensitive on the proper-case or upper-case form of each
|
||||
alias, with word boundaries. A one-word alias that is an English dictionary
|
||||
word ("Target", "Ball") is dropped unless listed in ALLOW, and the filer's
|
||||
own names are skipped.
|
||||
|
||||
A mention is counted in every role whose keywords appear in the same
|
||||
sentence: customer ("customer", "sales to", "revenue(s) from", "accounted
|
||||
for ... % of"), supplier ("supplier", "vendor", "supply", "purchase",
|
||||
"sole source", "foundry", "manufactur-"), competitor ("compet-", "rival").
|
||||
In filer i's 10-K, "sales to J accounted for 12% of revenue" marks J as a
|
||||
customer of i.
|
||||
|
||||
Output: data/research/sec_10k_links.csv, one row per (10-K, named company):
|
||||
filer, filed, accession, target, n_customer, n_supplier, n_competitor, n_all,
|
||||
example (first customer sentence, else the first sentence, 300 chars).
|
||||
data/research/sec_10k_links_done.txt lists processed accessions (resumable).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from research.lab import fetch_lazy_prices as lp
|
||||
|
||||
DIR = os.path.join("data", "research")
|
||||
OUT = os.path.join(DIR, "sec_10k_links.csv")
|
||||
DONE = os.path.join(DIR, "sec_10k_links_done.txt")
|
||||
NAMES = os.path.join(DIR, "sec_names.json")
|
||||
COLS = ["filer", "filed", "accession", "target", "n_customer", "n_supplier", "n_competitor", "n_all", "example"]
|
||||
|
||||
EXTRA = {"GOOGL": ["Google", "Alphabet"], "GOOG": ["Google", "Alphabet"], "META": ["Facebook", "Meta Platforms"],
|
||||
"AMZN": ["Amazon", "Amazon.com", "Amazon Web Services", "AWS"], "MSFT": ["Microsoft", "Azure"],
|
||||
"AAPL": ["Apple"], "HPQ": ["Hewlett-Packard", "HP Inc"], "HPE": ["Hewlett Packard Enterprise"],
|
||||
"WMT": ["Wal-Mart", "Walmart"], "IBM": ["IBM"], "AMD": ["AMD", "Advanced Micro Devices"],
|
||||
"NVDA": ["NVIDIA", "Nvidia"], "INTC": ["Intel"], "ORCL": ["Oracle"], "CSCO": ["Cisco"],
|
||||
"DELL": ["Dell", "Dell Technologies"], "GE": ["General Electric"], "TGT": ["Target Corporation"],
|
||||
"BA": ["Boeing"], "F": ["Ford Motor"], "GM": ["General Motors"], "TSLA": ["Tesla"],
|
||||
"CVS": ["CVS"], "UNH": ["UnitedHealth"], "MCK": ["McKesson"], "CAH": ["Cardinal Health"],
|
||||
"COR": ["AmerisourceBergen", "Cencora"], "HD": ["Home Depot"], "LOW": ["Lowe's", "Lowe’s"],
|
||||
"COST": ["Costco"], "KR": ["Kroger"], "T": ["AT&T"], "VZ": ["Verizon"], "TMUS": ["T-Mobile"],
|
||||
"CMCSA": ["Comcast"], "DIS": ["Disney"], "QCOM": ["Qualcomm", "QUALCOMM"], "AVGO": ["Broadcom"],
|
||||
"TXN": ["Texas Instruments"], "MU": ["Micron"], "WDC": ["Western Digital"], "STX": ["Seagate"],
|
||||
"LRCX": ["Lam Research"], "AMAT": ["Applied Materials"], "KLAC": ["KLA-Tencor", "KLA Corporation"],
|
||||
"SNDK": ["SanDisk", "Sandisk"], "GLW": ["Corning Incorporated"], "LMT": ["Lockheed Martin"],
|
||||
"RTX": ["Raytheon", "United Technologies"], "NOC": ["Northrop Grumman"], "GD": ["General Dynamics"],
|
||||
"UPS": ["UPS", "United Parcel Service"], "FDX": ["FedEx"], "BBY": ["Best Buy"], "EBAY": ["eBay"],
|
||||
"MMM": ["3M"], "COHR": ["Coherent Corp", "II-VI"], "BALL": ["Ball Corporation"],
|
||||
"PGR": ["Progressive Corporation"], "CRM": ["Salesforce", "salesforce.com"],
|
||||
"NDAQ": ["Nasdaq, Inc", "NASDAQ OMX"], "PFG": ["Principal Financial Group"], "DOW": ["Dow Chemical", "Dow Inc"]}
|
||||
# aliases that are mostly something else in a 10-K (the exchange, the officer title, the stock index)
|
||||
DROP = {"NDAQ": {"Nasdaq", "NASDAQ", "Nasdaq Stock Market", "NASDAQ STOCK MARKET"},
|
||||
"PFG": {"Principal Financial", "PRINCIPAL FINANCIAL"}, "DOW": {"Dow", "DOW"}}
|
||||
# sentences about people and financing documents, not business relationships
|
||||
NOISE = re.compile(r"served as\b|\bserves as\b|\bpositions? (?:with|at)\b|\broles? (?:with|at)\b|\bworked (?:at|for)\b|"
|
||||
r"\bboard of directors\b|\bprior to joining\b|\bChief \w+ Officer\b|\bVice President\b|\bIndenture\b|"
|
||||
r"\bCredit Agreement\b|\bRegistration Rights\b|\bunderwrit\w*\b|\btrustee\b|\bas (?:administrative )?agent\b|"
|
||||
r"\bRule 13a\b|\bCertification\b|\bExhibit\b|\bpar value\b|\bStock Market\b|\bSelect Market\b|"
|
||||
r"\bDow Jones\b|\bS&P\b", re.I)
|
||||
ALLOW = {"Apple", "Oracle", "Amazon", "Intel", "Micron", "Boeing", "Tesla", "Google", "Facebook", "Seagate",
|
||||
"Dell", "Cisco", "Comcast", "Disney", "Verizon", "Qualcomm", "Broadcom", "Microsoft", "Walmart",
|
||||
"Kroger", "Costco", "Nvidia", "McKesson", "Lumentum", "Corning", "Chevron", "Exxon", "Pfizer",
|
||||
"Merck", "Amgen", "Nike", "Starbucks", "Visa", "Mastercard", "Netflix", "Adobe", "Salesforce",
|
||||
"Alphabet"}
|
||||
SUFFIX = re.compile(r"(?:[\s,]+(?:inc|incorporated|corp|corporation|co|company|companies|ltd|limited|plc|llc|lp|l\.p|"
|
||||
r"holdings?|group|the|n\.v|s\.a|ag|se|de|del|new|\&))+\.?$", re.I)
|
||||
SLASH = re.compile(r"\s*/[A-Z]{2,}/?\s*$")
|
||||
KW = {"customer": re.compile(r"\bcustomers?\b|\bsales to\b|\brevenues? from\b|\baccounted for\b|\bnet sales to\b", re.I),
|
||||
"supplier": re.compile(r"\bsuppl(?:y|ier|iers|ied)\b|\bvendors?\b|\bpurchas\w*\b|\bsole[- ]source\b|\bfoundr(?:y|ies)\b|"
|
||||
r"\bmanufactur\w*\b", re.I),
|
||||
"competitor": re.compile(r"\bcompet\w*\b|\brivals?\b", re.I)}
|
||||
_SENT = re.compile(r"(?<=[.;!?])\s+(?=[A-Z(\"“])")
|
||||
|
||||
|
||||
def _clean_name(n: str) -> str:
|
||||
n = SLASH.sub("", n.strip())
|
||||
prev = None
|
||||
while prev != n:
|
||||
prev = n
|
||||
n = SUFFIX.sub("", n).strip(" ,.")
|
||||
return n
|
||||
|
||||
|
||||
def _variants(n: str) -> set[str]:
|
||||
out = {n}
|
||||
if n.isupper() and len(n) > 4:
|
||||
out.add(" ".join(w.capitalize() if len(w) > 3 or i == 0 else w.lower() if w in {"OF", "AND", "THE"} else w
|
||||
for i, w in enumerate(n.split())))
|
||||
if not n.isupper():
|
||||
out.add(n.upper())
|
||||
return {v for v in out if len(v) >= 3}
|
||||
|
||||
|
||||
def former_names(sic: pd.DataFrame) -> dict[str, list[str]]:
|
||||
if os.path.exists(NAMES):
|
||||
return json.load(open(NAMES))
|
||||
out = {}
|
||||
for cik in sic["cik"].dropna().astype(int).unique():
|
||||
r = lp._get(f"https://data.sec.gov/submissions/CIK{cik:010d}.json")
|
||||
if r is None:
|
||||
continue
|
||||
d = r.json()
|
||||
out[str(cik)] = [d.get("name", "")] + [f.get("name", "") for f in d.get("formerNames", [])]
|
||||
json.dump(out, open(NAMES, "w"))
|
||||
return out
|
||||
|
||||
|
||||
def aliases(sic: pd.DataFrame | None = None, fn: dict | None = None) -> dict[str, set[str]]:
|
||||
"""Name aliases per ticker. Defaults: the S&P 500 ever-members (sec_sic.csv,
|
||||
sec_names.json); the mid/small run passes the S&P 1500 frames."""
|
||||
if sic is None:
|
||||
sic = pd.read_csv(os.path.join(DIR, "sec_sic.csv"))
|
||||
if fn is None:
|
||||
fn = former_names(sic)
|
||||
wiki = pd.read_csv(os.path.join("data", "us_sectors.csv")).set_index("ticker")["Security"]
|
||||
words = {w.strip() for w in open("/usr/share/dict/words") if w.strip().islower()}
|
||||
out: dict[str, set[str]] = {}
|
||||
for r in sic.itertuples():
|
||||
names = list(fn.get(str(int(r.cik)), [])) + [r.name]
|
||||
if r.ticker in wiki.index:
|
||||
names.append(wiki[r.ticker])
|
||||
al = set()
|
||||
for n in names:
|
||||
if isinstance(n, str) and n.strip():
|
||||
al |= _variants(_clean_name(n))
|
||||
al |= set(EXTRA.get(r.ticker, []))
|
||||
al -= DROP.get(r.ticker, set())
|
||||
keep = set()
|
||||
for a in al:
|
||||
ws = a.split()
|
||||
if len(a) < 3:
|
||||
continue
|
||||
if len(ws) == 1 and (a.lower() in words or a.capitalize().lower() in words) and a not in ALLOW \
|
||||
and a.capitalize() not in ALLOW:
|
||||
continue
|
||||
if len(ws) == 1 and a.isupper() and len(a) <= 3 and a not in {"IBM", "AMD", "UPS", "CVS", "AWS", "HP"}:
|
||||
continue
|
||||
keep.add(a)
|
||||
out[r.ticker] = keep
|
||||
# an alias shared by two tickers (share classes, reused names) stays only if they are one company
|
||||
owner: dict[str, set[str]] = {}
|
||||
for t, al in out.items():
|
||||
for a in al:
|
||||
owner.setdefault(a, set()).add(t)
|
||||
same_co = [{"GOOGL", "GOOG"}, {"FOXA", "FOX"}, {"NWSA", "NWS"}, {"BRK-B", "BRK.B"}]
|
||||
for a, ts in owner.items():
|
||||
if len(ts) > 1 and not any(ts <= s for s in same_co):
|
||||
for t in ts:
|
||||
out[t].discard(a)
|
||||
return out
|
||||
|
||||
|
||||
_RX = None
|
||||
_ALIAS2T: dict[str, str] = {}
|
||||
|
||||
|
||||
def _compile(al: dict[str, set[str]]):
|
||||
global _RX, _ALIAS2T
|
||||
_ALIAS2T = {a: t for t, s in al.items() for a in s}
|
||||
alts = sorted(_ALIAS2T, key=len, reverse=True)
|
||||
_RX = re.compile(r"(?<![\w&-])(?:" + "|".join(re.escape(a) for a in alts) + r")(?![\w-])")
|
||||
|
||||
|
||||
def links_in(text: str, own: set[str]) -> dict[str, dict]:
|
||||
found: dict[str, dict] = {}
|
||||
for line in text.split("\n"):
|
||||
if len(line) < 20 or not _RX.search(line):
|
||||
continue
|
||||
for s in _SENT.split(line):
|
||||
hits = {_ALIAS2T[m.group(0)] for m in _RX.finditer(s)} - own
|
||||
if not hits or NOISE.search(s):
|
||||
continue
|
||||
roles = {k for k, rx in KW.items() if rx.search(s)}
|
||||
if "competitor" in roles: # "our competitors include X ... customers" is not a customer link
|
||||
roles = {"competitor"}
|
||||
for t in hits:
|
||||
d = found.setdefault(t, {"n_customer": 0, "n_supplier": 0, "n_competitor": 0, "n_all": 0, "example": ""})
|
||||
d["n_all"] += 1
|
||||
for k in roles:
|
||||
d[f"n_{k}"] += 1
|
||||
if (not d["example"]) or ("customer" in roles and d["n_customer"] == 1):
|
||||
d["example"] = s[:300]
|
||||
return found
|
||||
|
||||
|
||||
def work(job) -> tuple[str, list[dict]] | None:
|
||||
filer, own, cik, filed, acc, doc = job
|
||||
a = acc.replace("-", "")
|
||||
name = doc if isinstance(doc, str) and doc.strip() else f"{acc}.txt"
|
||||
r = lp._get(f"https://www.sec.gov/Archives/edgar/data/{cik}/{a}/{name}")
|
||||
if r is None:
|
||||
return acc, []
|
||||
if name.lower().endswith(".pdf"):
|
||||
return acc, []
|
||||
try:
|
||||
text = lp.clean_text(r.content)
|
||||
except Exception: # noqa: BLE001
|
||||
return acc, []
|
||||
rows = [{"filer": filer, "filed": filed, "accession": acc, "target": t, **d}
|
||||
for t, d in links_in(text, set(own)).items()]
|
||||
return acc, rows
|
||||
|
||||
|
||||
def _init(lock, nxt, al):
|
||||
lp._init_worker(lock, nxt)
|
||||
_compile(al)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# S&P 400 + 600 (research/lab/panel_mid.py): 10-Ks filed from 2015 on, names
|
||||
# matched against the S&P 1500 (the S&P 500 ever-members plus these)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MID_INDEX = os.path.join(DIR, "mid_10k_index.csv")
|
||||
MID_SIC = os.path.join(DIR, "mid_sec_sic.csv")
|
||||
MID_NAMES = os.path.join(DIR, "mid_sec_names.json")
|
||||
MID_OUT = os.path.join(DIR, "mid_10k_links.csv")
|
||||
MID_DONE = os.path.join(DIR, "mid_10k_links_done.txt")
|
||||
MID_START = "2015-01-01"
|
||||
|
||||
|
||||
def build_mid_index() -> None:
|
||||
"""CIKs of the mid/small panel's tickers (company_tickers.json, i.e. names
|
||||
still listed today, plus sec_sic.csv), their SEC names and 10-K filings."""
|
||||
from research.lab.fetch_us_fundamentals import cik_map
|
||||
from research.lab.panel_mid import load_us_mid
|
||||
tickers = [t for t in load_us_mid("midsmall").close.columns if t not in lp.US_ETFS]
|
||||
cm = cik_map()
|
||||
old = pd.read_csv(os.path.join(DIR, "sec_sic.csv")).set_index("ticker")["cik"].to_dict()
|
||||
tc = {t: int(old.get(t, cm.get(t, 0))) for t in tickers}
|
||||
tc = {t: c for t, c in tc.items() if c}
|
||||
print(f"{len(tc)}/{len(tickers)} mid/small tickers map to a CIK", flush=True)
|
||||
by_cik: dict[int, list[str]] = {}
|
||||
for t, c in tc.items():
|
||||
by_cik.setdefault(c, []).append(t)
|
||||
sic_rows, names, idx = [], {}, []
|
||||
t0 = time.time()
|
||||
for k, (cik, ts) in enumerate(by_cik.items(), 1):
|
||||
r = lp._get(f"https://data.sec.gov/submissions/CIK{cik:010d}.json")
|
||||
if r is None:
|
||||
continue
|
||||
js = r.json()
|
||||
names[str(cik)] = [js.get("name", "")] + [x.get("name", "") for x in js.get("formerNames", [])]
|
||||
parts = [lp._columns(js["filings"]["recent"])]
|
||||
for fl in js["filings"].get("files", []):
|
||||
if fl.get("filingTo", "9999") < MID_START:
|
||||
continue
|
||||
r2 = lp._get(f"https://data.sec.gov/submissions/{fl['name']}")
|
||||
if r2 is not None:
|
||||
parts.append(lp._columns(r2.json()))
|
||||
df = pd.concat(parts, ignore_index=True).drop_duplicates("accessionNumber")
|
||||
for t in sorted(ts):
|
||||
sic_rows.append({"ticker": t, "cik": cik, "sic": js.get("sic", ""),
|
||||
"sicDescription": js.get("sicDescription", ""), "name": js.get("name", "")})
|
||||
fk = df[df["form"].isin(["10-K", "10-K405"]) & (df["filingDate"] >= MID_START)]
|
||||
idx.append(pd.DataFrame({"cik": cik, "tickers": "|".join(sorted(ts)), "form": fk["form"],
|
||||
"filed": fk["filingDate"], "period": fk["reportDate"],
|
||||
"accession": fk["accessionNumber"], "doc": fk["primaryDocument"]}))
|
||||
if k % 100 == 0:
|
||||
print(f" submissions {k}/{len(by_cik)} [{time.time() - t0:.0f}s]", flush=True)
|
||||
pd.DataFrame(sic_rows).sort_values("ticker").to_csv(MID_SIC, index=False)
|
||||
json.dump(names, open(MID_NAMES, "w"))
|
||||
ix = pd.concat(idx, ignore_index=True).sort_values(["cik", "filed"])
|
||||
ix.to_csv(MID_INDEX, index=False)
|
||||
print(f"mid index: {len(ix)} 10-Ks for {ix['cik'].nunique()} CIKs [{time.time() - t0:.0f}s]", flush=True)
|
||||
|
||||
|
||||
def sp1500_frames() -> tuple[pd.DataFrame, dict]:
|
||||
big = pd.read_csv(os.path.join(DIR, "sec_sic.csv"))
|
||||
mid = pd.read_csv(MID_SIC)
|
||||
sic = pd.concat([big, mid[~mid["ticker"].isin(big["ticker"])]], ignore_index=True)
|
||||
fn = {**json.load(open(MID_NAMES)), **json.load(open(NAMES))}
|
||||
return sic, fn
|
||||
|
||||
|
||||
def main() -> None:
|
||||
global OUT, DONE
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--workers", type=int, default=4)
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--universe", default="sp500", choices=["sp500", "mid"])
|
||||
args = ap.parse_args()
|
||||
lock, nxt = mp.Lock(), mp.Value("d", 0.0)
|
||||
lp._init_worker(lock, nxt)
|
||||
if args.universe == "mid":
|
||||
if not os.path.exists(MID_INDEX):
|
||||
build_mid_index()
|
||||
OUT, DONE = MID_OUT, MID_DONE
|
||||
sic_all, fn = sp1500_frames()
|
||||
al = aliases(sic_all, fn)
|
||||
ix = pd.read_csv(MID_INDEX, dtype={"doc": str})
|
||||
sic = pd.read_csv(MID_SIC)
|
||||
else:
|
||||
al = aliases()
|
||||
ix = pd.read_csv(os.path.join(DIR, "lazy_prices_index.csv"), dtype={"doc": str})
|
||||
sic = pd.read_csv(os.path.join(DIR, "sec_sic.csv"))
|
||||
print(f"{len(al)} tickers, {sum(map(len, al.values()))} aliases", flush=True)
|
||||
ix = ix[ix["form"].isin(["10-K", "10-K405"])]
|
||||
cik2t = sic.groupby("cik")["ticker"].apply(list).to_dict()
|
||||
done = set(open(DONE).read().split()) if os.path.exists(DONE) else set()
|
||||
jobs = []
|
||||
for r in ix.itertuples():
|
||||
if r.accession in done:
|
||||
continue
|
||||
ts = cik2t.get(r.cik, str(r.tickers).split("|"))
|
||||
jobs.append((ts[0], [t for t in ts], int(r.cik), r.filed, r.accession, r.doc))
|
||||
if args.limit:
|
||||
jobs = jobs[:args.limit]
|
||||
print(f"{len(jobs)} 10-Ks to scan ({len(done)} done)", flush=True)
|
||||
if not os.path.exists(OUT):
|
||||
with open(OUT, "w", newline="") as fh:
|
||||
csv.DictWriter(fh, fieldnames=COLS).writeheader()
|
||||
t0 = time.time()
|
||||
with mp.get_context("fork").Pool(args.workers, initializer=_init, initargs=(lock, nxt, al)) as pool, \
|
||||
open(OUT, "a", newline="") as out, open(DONE, "a") as dn:
|
||||
w = csv.DictWriter(out, fieldnames=COLS)
|
||||
for k, (acc, rows) in enumerate(pool.imap_unordered(work, jobs, chunksize=4), 1):
|
||||
for row in rows:
|
||||
w.writerow(row)
|
||||
dn.write(acc + "\n")
|
||||
if k % 200 == 0 or k == len(jobs):
|
||||
out.flush(); dn.flush()
|
||||
el = time.time() - t0
|
||||
print(f" {k}/{len(jobs)} [{el/60:.0f} min, ETA {(len(jobs)-k)*el/k/60:.0f} min]", flush=True)
|
||||
print("done", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user