""" Economic links between S&P 500 ever-members from their 10-K filings: which other member companies each 10-K names, and in what role (customer, supplier, competitor). The links become known on the filing date, so a signal built from them is point-in-time. PYTHONPATH=. uv run python -m research.lab.fetch_supply_links --workers 4 PYTHONPATH=. uv run python -m research.lab.fetch_supply_links --universe mid # S&P 400/600 10-Ks from 2015 Inputs: data/research/lazy_prices_index.csv (10-K accessions and primary documents, from research/lab/fetch_lazy_prices.py) and sec_sic.csv (ticker, CIK, current SEC name). Names: current and former SEC names (submissions JSON, cached in data/research/sec_names.json), the Wikipedia security name of current members, and a short manual list of brands (Google, Facebook, AWS). Matching is case-sensitive on the proper-case or upper-case form of each alias, with word boundaries. A one-word alias that is an English dictionary word ("Target", "Ball") is dropped unless listed in ALLOW, and the filer's own names are skipped. A mention is counted in every role whose keywords appear in the same sentence: customer ("customer", "sales to", "revenue(s) from", "accounted for ... % of"), supplier ("supplier", "vendor", "supply", "purchase", "sole source", "foundry", "manufactur-"), competitor ("compet-", "rival"). In filer i's 10-K, "sales to J accounted for 12% of revenue" marks J as a customer of i. Output: data/research/sec_10k_links.csv, one row per (10-K, named company): filer, filed, accession, target, n_customer, n_supplier, n_competitor, n_all, example (first customer sentence, else the first sentence, 300 chars). data/research/sec_10k_links_done.txt lists processed accessions (resumable). """ from __future__ import annotations import argparse import csv import json import multiprocessing as mp import os import re import time import pandas as pd from research.lab import fetch_lazy_prices as lp DIR = os.path.join("data", "research") OUT = os.path.join(DIR, "sec_10k_links.csv") DONE = os.path.join(DIR, "sec_10k_links_done.txt") NAMES = os.path.join(DIR, "sec_names.json") COLS = ["filer", "filed", "accession", "target", "n_customer", "n_supplier", "n_competitor", "n_all", "example"] EXTRA = {"GOOGL": ["Google", "Alphabet"], "GOOG": ["Google", "Alphabet"], "META": ["Facebook", "Meta Platforms"], "AMZN": ["Amazon", "Amazon.com", "Amazon Web Services", "AWS"], "MSFT": ["Microsoft", "Azure"], "AAPL": ["Apple"], "HPQ": ["Hewlett-Packard", "HP Inc"], "HPE": ["Hewlett Packard Enterprise"], "WMT": ["Wal-Mart", "Walmart"], "IBM": ["IBM"], "AMD": ["AMD", "Advanced Micro Devices"], "NVDA": ["NVIDIA", "Nvidia"], "INTC": ["Intel"], "ORCL": ["Oracle"], "CSCO": ["Cisco"], "DELL": ["Dell", "Dell Technologies"], "GE": ["General Electric"], "TGT": ["Target Corporation"], "BA": ["Boeing"], "F": ["Ford Motor"], "GM": ["General Motors"], "TSLA": ["Tesla"], "CVS": ["CVS"], "UNH": ["UnitedHealth"], "MCK": ["McKesson"], "CAH": ["Cardinal Health"], "COR": ["AmerisourceBergen", "Cencora"], "HD": ["Home Depot"], "LOW": ["Lowe's", "Lowe’s"], "COST": ["Costco"], "KR": ["Kroger"], "T": ["AT&T"], "VZ": ["Verizon"], "TMUS": ["T-Mobile"], "CMCSA": ["Comcast"], "DIS": ["Disney"], "QCOM": ["Qualcomm", "QUALCOMM"], "AVGO": ["Broadcom"], "TXN": ["Texas Instruments"], "MU": ["Micron"], "WDC": ["Western Digital"], "STX": ["Seagate"], "LRCX": ["Lam Research"], "AMAT": ["Applied Materials"], "KLAC": ["KLA-Tencor", "KLA Corporation"], "SNDK": ["SanDisk", "Sandisk"], "GLW": ["Corning Incorporated"], "LMT": ["Lockheed Martin"], "RTX": ["Raytheon", "United Technologies"], "NOC": ["Northrop Grumman"], "GD": ["General Dynamics"], "UPS": ["UPS", "United Parcel Service"], "FDX": ["FedEx"], "BBY": ["Best Buy"], "EBAY": ["eBay"], "MMM": ["3M"], "COHR": ["Coherent Corp", "II-VI"], "BALL": ["Ball Corporation"], "PGR": ["Progressive Corporation"], "CRM": ["Salesforce", "salesforce.com"], "NDAQ": ["Nasdaq, Inc", "NASDAQ OMX"], "PFG": ["Principal Financial Group"], "DOW": ["Dow Chemical", "Dow Inc"]} # aliases that are mostly something else in a 10-K (the exchange, the officer title, the stock index) DROP = {"NDAQ": {"Nasdaq", "NASDAQ", "Nasdaq Stock Market", "NASDAQ STOCK MARKET"}, "PFG": {"Principal Financial", "PRINCIPAL FINANCIAL"}, "DOW": {"Dow", "DOW"}} # sentences about people and financing documents, not business relationships NOISE = re.compile(r"served as\b|\bserves as\b|\bpositions? (?:with|at)\b|\broles? (?:with|at)\b|\bworked (?:at|for)\b|" r"\bboard of directors\b|\bprior to joining\b|\bChief \w+ Officer\b|\bVice President\b|\bIndenture\b|" r"\bCredit Agreement\b|\bRegistration Rights\b|\bunderwrit\w*\b|\btrustee\b|\bas (?:administrative )?agent\b|" r"\bRule 13a\b|\bCertification\b|\bExhibit\b|\bpar value\b|\bStock Market\b|\bSelect Market\b|" r"\bDow Jones\b|\bS&P\b", re.I) ALLOW = {"Apple", "Oracle", "Amazon", "Intel", "Micron", "Boeing", "Tesla", "Google", "Facebook", "Seagate", "Dell", "Cisco", "Comcast", "Disney", "Verizon", "Qualcomm", "Broadcom", "Microsoft", "Walmart", "Kroger", "Costco", "Nvidia", "McKesson", "Lumentum", "Corning", "Chevron", "Exxon", "Pfizer", "Merck", "Amgen", "Nike", "Starbucks", "Visa", "Mastercard", "Netflix", "Adobe", "Salesforce", "Alphabet"} SUFFIX = re.compile(r"(?:[\s,]+(?:inc|incorporated|corp|corporation|co|company|companies|ltd|limited|plc|llc|lp|l\.p|" r"holdings?|group|the|n\.v|s\.a|ag|se|de|del|new|\&))+\.?$", re.I) SLASH = re.compile(r"\s*/[A-Z]{2,}/?\s*$") KW = {"customer": re.compile(r"\bcustomers?\b|\bsales to\b|\brevenues? from\b|\baccounted for\b|\bnet sales to\b", re.I), "supplier": re.compile(r"\bsuppl(?:y|ier|iers|ied)\b|\bvendors?\b|\bpurchas\w*\b|\bsole[- ]source\b|\bfoundr(?:y|ies)\b|" r"\bmanufactur\w*\b", re.I), "competitor": re.compile(r"\bcompet\w*\b|\brivals?\b", re.I)} _SENT = re.compile(r"(?<=[.;!?])\s+(?=[A-Z(\"“])") def _clean_name(n: str) -> str: n = SLASH.sub("", n.strip()) prev = None while prev != n: prev = n n = SUFFIX.sub("", n).strip(" ,.") return n def _variants(n: str) -> set[str]: out = {n} if n.isupper() and len(n) > 4: out.add(" ".join(w.capitalize() if len(w) > 3 or i == 0 else w.lower() if w in {"OF", "AND", "THE"} else w for i, w in enumerate(n.split()))) if not n.isupper(): out.add(n.upper()) return {v for v in out if len(v) >= 3} def former_names(sic: pd.DataFrame) -> dict[str, list[str]]: if os.path.exists(NAMES): return json.load(open(NAMES)) out = {} for cik in sic["cik"].dropna().astype(int).unique(): r = lp._get(f"https://data.sec.gov/submissions/CIK{cik:010d}.json") if r is None: continue d = r.json() out[str(cik)] = [d.get("name", "")] + [f.get("name", "") for f in d.get("formerNames", [])] json.dump(out, open(NAMES, "w")) return out def aliases(sic: pd.DataFrame | None = None, fn: dict | None = None) -> dict[str, set[str]]: """Name aliases per ticker. Defaults: the S&P 500 ever-members (sec_sic.csv, sec_names.json); the mid/small run passes the S&P 1500 frames.""" if sic is None: sic = pd.read_csv(os.path.join(DIR, "sec_sic.csv")) if fn is None: fn = former_names(sic) wiki = pd.read_csv(os.path.join("data", "us_sectors.csv")).set_index("ticker")["Security"] words = {w.strip() for w in open("/usr/share/dict/words") if w.strip().islower()} out: dict[str, set[str]] = {} for r in sic.itertuples(): names = list(fn.get(str(int(r.cik)), [])) + [r.name] if r.ticker in wiki.index: names.append(wiki[r.ticker]) al = set() for n in names: if isinstance(n, str) and n.strip(): al |= _variants(_clean_name(n)) al |= set(EXTRA.get(r.ticker, [])) al -= DROP.get(r.ticker, set()) keep = set() for a in al: ws = a.split() if len(a) < 3: continue if len(ws) == 1 and (a.lower() in words or a.capitalize().lower() in words) and a not in ALLOW \ and a.capitalize() not in ALLOW: continue if len(ws) == 1 and a.isupper() and len(a) <= 3 and a not in {"IBM", "AMD", "UPS", "CVS", "AWS", "HP"}: continue keep.add(a) out[r.ticker] = keep # an alias shared by two tickers (share classes, reused names) stays only if they are one company owner: dict[str, set[str]] = {} for t, al in out.items(): for a in al: owner.setdefault(a, set()).add(t) same_co = [{"GOOGL", "GOOG"}, {"FOXA", "FOX"}, {"NWSA", "NWS"}, {"BRK-B", "BRK.B"}] for a, ts in owner.items(): if len(ts) > 1 and not any(ts <= s for s in same_co): for t in ts: out[t].discard(a) return out _RX = None _ALIAS2T: dict[str, str] = {} def _compile(al: dict[str, set[str]]): global _RX, _ALIAS2T _ALIAS2T = {a: t for t, s in al.items() for a in s} alts = sorted(_ALIAS2T, key=len, reverse=True) _RX = re.compile(r"(? dict[str, dict]: found: dict[str, dict] = {} for line in text.split("\n"): if len(line) < 20 or not _RX.search(line): continue for s in _SENT.split(line): hits = {_ALIAS2T[m.group(0)] for m in _RX.finditer(s)} - own if not hits or NOISE.search(s): continue roles = {k for k, rx in KW.items() if rx.search(s)} if "competitor" in roles: # "our competitors include X ... customers" is not a customer link roles = {"competitor"} for t in hits: d = found.setdefault(t, {"n_customer": 0, "n_supplier": 0, "n_competitor": 0, "n_all": 0, "example": ""}) d["n_all"] += 1 for k in roles: d[f"n_{k}"] += 1 if (not d["example"]) or ("customer" in roles and d["n_customer"] == 1): d["example"] = s[:300] return found def work(job) -> tuple[str, list[dict]] | None: filer, own, cik, filed, acc, doc = job a = acc.replace("-", "") name = doc if isinstance(doc, str) and doc.strip() else f"{acc}.txt" r = lp._get(f"https://www.sec.gov/Archives/edgar/data/{cik}/{a}/{name}") if r is None: return acc, [] if name.lower().endswith(".pdf"): return acc, [] try: text = lp.clean_text(r.content) except Exception: # noqa: BLE001 return acc, [] rows = [{"filer": filer, "filed": filed, "accession": acc, "target": t, **d} for t, d in links_in(text, set(own)).items()] return acc, rows def _init(lock, nxt, al): lp._init_worker(lock, nxt) _compile(al) # --------------------------------------------------------------------------- # S&P 400 + 600 (research/lab/panel_mid.py): 10-Ks filed from 2015 on, names # matched against the S&P 1500 (the S&P 500 ever-members plus these) # --------------------------------------------------------------------------- MID_INDEX = os.path.join(DIR, "mid_10k_index.csv") MID_SIC = os.path.join(DIR, "mid_sec_sic.csv") MID_NAMES = os.path.join(DIR, "mid_sec_names.json") MID_OUT = os.path.join(DIR, "mid_10k_links.csv") MID_DONE = os.path.join(DIR, "mid_10k_links_done.txt") MID_START = "2015-01-01" def build_mid_index() -> None: """CIKs of the mid/small panel's tickers (company_tickers.json, i.e. names still listed today, plus sec_sic.csv), their SEC names and 10-K filings.""" from research.lab.fetch_us_fundamentals import cik_map from research.lab.panel_mid import load_us_mid tickers = [t for t in load_us_mid("midsmall").close.columns if t not in lp.US_ETFS] cm = cik_map() old = pd.read_csv(os.path.join(DIR, "sec_sic.csv")).set_index("ticker")["cik"].to_dict() tc = {t: int(old.get(t, cm.get(t, 0))) for t in tickers} tc = {t: c for t, c in tc.items() if c} print(f"{len(tc)}/{len(tickers)} mid/small tickers map to a CIK", flush=True) by_cik: dict[int, list[str]] = {} for t, c in tc.items(): by_cik.setdefault(c, []).append(t) sic_rows, names, idx = [], {}, [] t0 = time.time() for k, (cik, ts) in enumerate(by_cik.items(), 1): r = lp._get(f"https://data.sec.gov/submissions/CIK{cik:010d}.json") if r is None: continue js = r.json() names[str(cik)] = [js.get("name", "")] + [x.get("name", "") for x in js.get("formerNames", [])] parts = [lp._columns(js["filings"]["recent"])] for fl in js["filings"].get("files", []): if fl.get("filingTo", "9999") < MID_START: continue r2 = lp._get(f"https://data.sec.gov/submissions/{fl['name']}") if r2 is not None: parts.append(lp._columns(r2.json())) df = pd.concat(parts, ignore_index=True).drop_duplicates("accessionNumber") for t in sorted(ts): sic_rows.append({"ticker": t, "cik": cik, "sic": js.get("sic", ""), "sicDescription": js.get("sicDescription", ""), "name": js.get("name", "")}) fk = df[df["form"].isin(["10-K", "10-K405"]) & (df["filingDate"] >= MID_START)] idx.append(pd.DataFrame({"cik": cik, "tickers": "|".join(sorted(ts)), "form": fk["form"], "filed": fk["filingDate"], "period": fk["reportDate"], "accession": fk["accessionNumber"], "doc": fk["primaryDocument"]})) if k % 100 == 0: print(f" submissions {k}/{len(by_cik)} [{time.time() - t0:.0f}s]", flush=True) pd.DataFrame(sic_rows).sort_values("ticker").to_csv(MID_SIC, index=False) json.dump(names, open(MID_NAMES, "w")) ix = pd.concat(idx, ignore_index=True).sort_values(["cik", "filed"]) ix.to_csv(MID_INDEX, index=False) print(f"mid index: {len(ix)} 10-Ks for {ix['cik'].nunique()} CIKs [{time.time() - t0:.0f}s]", flush=True) def sp1500_frames() -> tuple[pd.DataFrame, dict]: big = pd.read_csv(os.path.join(DIR, "sec_sic.csv")) mid = pd.read_csv(MID_SIC) sic = pd.concat([big, mid[~mid["ticker"].isin(big["ticker"])]], ignore_index=True) fn = {**json.load(open(MID_NAMES)), **json.load(open(NAMES))} return sic, fn def main() -> None: global OUT, DONE ap = argparse.ArgumentParser() ap.add_argument("--workers", type=int, default=4) ap.add_argument("--limit", type=int, default=0) ap.add_argument("--universe", default="sp500", choices=["sp500", "mid"]) args = ap.parse_args() lock, nxt = mp.Lock(), mp.Value("d", 0.0) lp._init_worker(lock, nxt) if args.universe == "mid": if not os.path.exists(MID_INDEX): build_mid_index() OUT, DONE = MID_OUT, MID_DONE sic_all, fn = sp1500_frames() al = aliases(sic_all, fn) ix = pd.read_csv(MID_INDEX, dtype={"doc": str}) sic = pd.read_csv(MID_SIC) else: al = aliases() ix = pd.read_csv(os.path.join(DIR, "lazy_prices_index.csv"), dtype={"doc": str}) sic = pd.read_csv(os.path.join(DIR, "sec_sic.csv")) print(f"{len(al)} tickers, {sum(map(len, al.values()))} aliases", flush=True) ix = ix[ix["form"].isin(["10-K", "10-K405"])] cik2t = sic.groupby("cik")["ticker"].apply(list).to_dict() done = set(open(DONE).read().split()) if os.path.exists(DONE) else set() jobs = [] for r in ix.itertuples(): if r.accession in done: continue ts = cik2t.get(r.cik, str(r.tickers).split("|")) jobs.append((ts[0], [t for t in ts], int(r.cik), r.filed, r.accession, r.doc)) if args.limit: jobs = jobs[:args.limit] print(f"{len(jobs)} 10-Ks to scan ({len(done)} done)", flush=True) if not os.path.exists(OUT): with open(OUT, "w", newline="") as fh: csv.DictWriter(fh, fieldnames=COLS).writeheader() t0 = time.time() with mp.get_context("fork").Pool(args.workers, initializer=_init, initargs=(lock, nxt, al)) as pool, \ open(OUT, "a", newline="") as out, open(DONE, "a") as dn: w = csv.DictWriter(out, fieldnames=COLS) for k, (acc, rows) in enumerate(pool.imap_unordered(work, jobs, chunksize=4), 1): for row in rows: w.writerow(row) dn.write(acc + "\n") if k % 200 == 0 or k == len(jobs): out.flush(); dn.flush() el = time.time() - t0 print(f" {k}/{len(jobs)} [{el/60:.0f} min, ETA {(len(jobs)-k)*el/k/60:.0f} min]", flush=True) print("done", flush=True) if __name__ == "__main__": main()