""" Can sector cyclicality be modelled well enough to improve the US book? Business-cycle sector rotation, sector momentum and reversal, cycle-position signals, and cycle-conditioned versions of the momentum sleeve. PYTHONPATH=. uv run python -m research.lab.sector_cycle fetch # FRED series + sector ETFs PYTHONPATH=. uv run python -m research.lab.sector_cycle # all tests Yardstick (as in beat_qqq.py): alpha against QQQ and SPY (Newey-West t), design 2006-2016 vs holdout 2017-2026. ETF-level tests also run from 1999 and are split at 2008. Every variant is fixed before it is run and logged in data/research/results/sector_cycle_trials.csv. Inputs and their lags sector ETFs the nine 1998 SPDRs (XLB XLE XLF XLI XLK XLP XLU XLV XLY), Yahoo adjusted closes. XLC (2018) and XLRE (2015) are too short. GICS sector p.extra["sector"]: today's classification, not point in time. SIC 2-digit data/research/sec_sic.csv: today's code, not point in time. FRED a monthly value for month m (CFNAI, INDPRO, UNRATE, CPIAUCSL) is used from the last trading day of m+1 (each is released before then); weekly ICSA 7 days after the week's date; daily T10Y3M and BAA10Y the next business day. Latest vintage only: revisions are not modelled (ALFRED vintages were not reachable), which flatters CFNAI and INDPRO. ISM PMI (NAPM) is no longer on FRED. Tests (a) Business-cycle rotation. Phase by the OECD four-phase rule (below/above trend x rising/falling) on CFNAI-MA3 (trend = 0, direction = 3-month change) and on INDPRO 12-month log growth (trend = its trailing 10-year mean). Phase -> favoured sectors from Fidelity's business-cycle approach, not fitted. (b) Sector momentum: cross-sectional 12-1 top 3, time-series 12-month (a sector is held only while it beat T-bills over 12 months), long-horizon reversal (bottom 3 by months 60..13). ETFs and EW PIT GICS sectors. (c) Cycle position: price position in the 3-year range, rank IC against the next 3/6/12 months across ETFs, GICS sectors and SIC-2 industries; EDGAR revenue-growth level and acceleration by sector (2011+); the semiconductor cycle (^SOX from 1994) with position, relative momentum and SIC 3674 revenue acceleration. (d) The momentum sleeve (overnight + rec_126, top 10, every 10 days, buffer 2 = overnight_recovery_top10) by macro state; a cash switch in the state picked on 2006-2016 only; its picks restricted to the CFNAI phase's favoured sectors, or with the phase's unfavoured sectors removed. """ from __future__ import annotations import functools import json import os import sys import warnings import numpy as np import pandas as pd from research.lab.alpha import alpha_stats, bench_returns from research.lab.factor_scan import RESULTS from research.lab.levered_book import tbill from research.lab.report import stats from research.lab.us_final import monthly_blend warnings.filterwarnings("ignore") EXT = os.path.join("data", "research", "ext") YAHOO = os.path.join(EXT, "yahoo") SIC = os.path.join("data", "research", "sec_sic.csv") ETF9 = ["XLB", "XLE", "XLF", "XLI", "XLK", "XLP", "XLU", "XLV", "XLY"] FRED_SERIES = ["CFNAI", "INDPRO", "UNRATE", "CPIAUCSL", "ICSA", "T10Y3M", "BAA10Y", "USREC"] MONTHLY = {"CFNAI", "INDPRO", "UNRATE", "CPIAUCSL", "USREC"} LONG = ("2006-09-27", "2026-09-24") SPANS = {"full": (None, None), "design": ("2006-01-01", "2016-12-31"), "holdout": ("2017-01-01", None), "pre08": (None, "2007-12-31"), "post08": ("2008-01-01", None)} PHASES = ["early", "mid", "late", "recession"] # Fidelity, "The Business Cycle Approach to Equity Sector Investing": sectors # that have tended to beat / trail the market in each phase. FAVOURED = { "early": ["Financials", "Real Estate", "Consumer Discretionary", "Industrials", "Information Technology", "Materials"], "mid": ["Information Technology", "Communication Services", "Industrials"], "late": ["Energy", "Materials", "Consumer Staples", "Health Care", "Utilities"], "recession": ["Consumer Staples", "Utilities", "Health Care"], } UNFAVOURED = { "early": ["Consumer Staples", "Energy", "Health Care", "Communication Services", "Utilities"], "mid": ["Materials", "Utilities"], "late": ["Information Technology", "Consumer Discretionary"], "recession": ["Industrials", "Information Technology", "Real Estate"], } ETF_SECTOR = {"XLB": "Materials", "XLE": "Energy", "XLF": "Financials", "XLI": "Industrials", "XLK": "Information Technology", "XLP": "Consumer Staples", "XLU": "Utilities", "XLV": "Health Care", "XLY": "Consumer Discretionary"} TRIALS: list[dict] = [] # --------------------------------------------------------------------------- # data # --------------------------------------------------------------------------- def fetch() -> None: from research.lab.fetch_long_history import fetch_fred, fetch_yahoo df = fetch_fred(FRED_SERIES) for s in FRED_SERIES: df[[s]].dropna().rename_axis("observation_date").to_csv(os.path.join(EXT, f"fred_{s}.csv")) print(s, df[s].dropna().index[0].date(), df[s].dropna().index[-1].date()) fetch_yahoo(ETF9 + ["XLC", "XLRE", "^SOX", "SMH", "SOXX"], start="1990-01-01") def fred(s: str) -> pd.Series: x = pd.read_csv(os.path.join(EXT, f"fred_{s}.csv"), index_col=0, parse_dates=True).iloc[:, 0] return pd.to_numeric(x, errors="coerce").dropna() def available(x: pd.Series, s: str) -> pd.Series: """Re-index a FRED series (or a transform of it) by the date it is known.""" if s in MONTHLY: idx = x.index + pd.offsets.MonthEnd(2) # obs dated m-01 -> end of m+1 elif s == "ICSA": idx = x.index + pd.Timedelta(days=7) else: idx = x.index + pd.offsets.BDay(1) return pd.Series(x.values, index=idx).groupby(level=0).last() def asof(x: pd.Series, dates: pd.DatetimeIndex) -> pd.Series: return x.reindex(dates.union(x.index)).ffill().reindex(dates) @functools.lru_cache def yahoo(t: str) -> pd.Series: path = os.path.join(YAHOO, t.replace("^", "_").replace("=", "_") + ".csv") return pd.read_csv(path, index_col=0, parse_dates=True)["Close"] def etf_prices(tickers=ETF9) -> pd.DataFrame: return pd.DataFrame({t: yahoo(t) for t in tickers}).sort_index() def month_ends(dates: pd.DatetimeIndex) -> pd.DatetimeIndex: s = pd.Series(dates, index=dates) return pd.DatetimeIndex(s.groupby(dates.to_period("M")).max().values) # --------------------------------------------------------------------------- # macro phases and states (index = date the value is known) # --------------------------------------------------------------------------- def _four_phase(level_above: pd.Series, rising: pd.Series) -> pd.Series: ph = np.select([~level_above & rising, level_above & rising, level_above & ~rising, ~level_above & ~rising], PHASES, default=None) return pd.Series(ph, index=level_above.index) def phase_cfnai(raw: bool = False) -> pd.Series: """raw=True: indexed by the observation month (look-ahead; for the oracle bound and the descriptive table only).""" m = fred("CFNAI").rolling(3).mean().dropna() d = m.diff(3) ok = d.notna() ph = _four_phase(m[ok] > 0, d[ok] > 0) return ph if raw else available(ph, "CFNAI") def phase_indpro() -> pd.Series: g = np.log(fred("INDPRO")).diff(12) trend = g.rolling(120, min_periods=60).mean() d = g.diff(3) ok = g.notna() & trend.notna() & d.notna() return available(_four_phase((g > trend)[ok], (d > 0)[ok]), "INDPRO") def states(dates: pd.DatetimeIndex) -> pd.DataFrame: """Macro states known at each date (strings / bools).""" out = pd.DataFrame(index=dates) out["cfnai"] = asof(phase_cfnai(), dates) out["indpro"] = asof(phase_indpro(), dates) out["curve_inverted"] = asof(available(fred("T10Y3M") < 0, "T10Y3M"), dates) baa = fred("BAA10Y") out["credit_widening"] = asof(available(baa - baa.shift(63) > 0, "BAA10Y").astype(float), dates) > 0 ic = fred("ICSA").rolling(13).mean() out["claims_rising"] = asof(available(ic > ic.shift(26), "ICSA").astype(float), dates) > 0 u = fred("UNRATE").rolling(3).mean() out["sahm"] = asof(available(u - u.rolling(12).min().shift(1) >= 0.5, "UNRATE").astype(float), dates) > 0 cpi = np.log(fred("CPIAUCSL")).diff(12) out["inflation_rising"] = asof(available(cpi.diff(3) > 0, "CPIAUCSL").astype(float), dates) > 0 return out # --------------------------------------------------------------------------- # books # --------------------------------------------------------------------------- def etf_book(W: pd.DataFrame, px: pd.DataFrame, cost: float = 0.001) -> pd.Series: """Monthly ETF book. W holds target weights on signal dates (rows sum to at most 1, the rest earns T-bills); a signal is traded at the next day's close, and holdings drift between rebalances.""" px = px[W.columns].ffill() dates = px.index ret = px.pct_change(fill_method=None).fillna(0.0).to_numpy() rf = (tbill(dates).fillna(0.0) / 252).to_numpy() T = W.reindex(dates).to_numpy() valid = ~np.isnan(T).all(axis=1) first = int(np.argmax(valid)) + 1 h = np.zeros(px.shape[1]); cash = 1.0 eq = np.full(len(dates), np.nan); eq[first - 1] = 1.0 for i in range(first, len(dates)): h = h * (1 + ret[i]); cash *= 1 + rf[i] V = h.sum() + cash if valid[i - 1]: tgt = np.nan_to_num(T[i - 1]) * V c = np.abs(tgt - h).sum() * cost V -= c h = np.nan_to_num(T[i - 1]) * V cash = V - h.sum() eq[i] = V return pd.Series(eq, index=dates).pct_change().dropna() def stock_book(p, W: pd.DataFrame) -> pd.Series: """Monthly book of stocks from explicit target weights (engine.simulate, next-open fills, 10 bp, PIT members).""" from research.lab.engine import simulate sp = p.slice("2005-01-01", LONG[1]) r = simulate(sp, target_weights=W.reindex(columns=sp.close.columns), every="M", exec_at="open") first = W.dropna(how="all").index[0] return r.returns.loc[first:].iloc[1:] def sector_weights(chosen: dict, members: pd.DataFrame, sector: pd.Series) -> pd.DataFrame: """chosen: {signal date: {sector: weight}} -> equal weight within each sector.""" rows = {} for d, sw in chosen.items(): m = members.loc[d] w = pd.Series(0.0, index=members.columns) for s, x in sw.items(): cols = m.index[m & (sector == s)] if len(cols) and x > 0: w[cols] = x / len(cols) rows[d] = w return pd.DataFrame(rows).T # --------------------------------------------------------------------------- # evaluation # --------------------------------------------------------------------------- @functools.lru_cache def momentum_sleeve() -> pd.Series: return pd.read_pickle(os.path.join(RESULTS, "beat_qqq_sleeves.pkl"))["momentum"] def _mdd(r: pd.Series) -> float: e = (1 + r).cumprod() return float((e / e.cummax() - 1).min()) def evaluate(name: str, r: pd.Series, test: str, ew: pd.Series | None = None, log: bool = True) -> dict: r = r.dropna() out = {"name": name, "test": test, "start": str(r.index[0].date())} for k, (a, b) in SPANS.items(): x = r.loc[a:b] if len(x) < 250: continue q = alpha_stats(x, bench_returns(x.index, "QQQ")) s = alpha_stats(x, bench_returns(x.index, "SPY")) st = stats((1 + x).cumprod()) row = {"alpha_qqq": q["alpha"], "t_qqq": q["t"], "beta_qqq": q["beta"], "alpha_spy": s["alpha"], "t_spy": s["t"], "cagr": st["cagr"], "sharpe": st["sharpe"], "mdd": _mdd(x)} if ew is not None: e = alpha_stats(x, ew.reindex(x.index)) row.update(alpha_ew=e["alpha"], t_ew=e["t"]) out[k] = row mom = momentum_sleeve() j = pd.concat({"x": r, "m": mom}, axis=1).dropna() if len(j) > 500: mm = (1 + j).resample("ME").prod() - 1 out["corr_mom"] = float(mm.corr().iloc[0, 1]) bl = monthly_blend(j, {"x": 0.5, "m": 0.5}) inc = {} for k in ("full", "design", "holdout"): a, b = SPANS[k] xb, xm = bl.loc[a:b], j["m"].loc[a:b] qb = alpha_stats(xb, bench_returns(xb.index, "QQQ")) qm = alpha_stats(xm, bench_returns(xm.index, "QQQ")) inc[k] = {"blend_sharpe": stats((1 + xb).cumprod())["sharpe"], "mom_sharpe": stats((1 + xm).cumprod())["sharpe"], "blend_alpha_qqq": qb["alpha"], "blend_t_qqq": qb["t"], "mom_alpha_qqq": qm["alpha"], "blend_mdd": _mdd(xb), "mom_mdd": _mdd(xm)} out["blend50"] = inc if log: TRIALS.append(out) print(line(out), flush=True) return out def line(o: dict) -> str: def cell(k, key="alpha_qqq", t="t_qqq"): if k not in o: return " -- " return f"{o[k][key]*100:6.1f}% t{o[k][t]:5.2f}" f = o.get("full", {}) s = (f"{o['name']:44s} QQQ {cell('full')} | des {cell('design')} | hold {cell('holdout')} | " f"SPY {cell('full', 'alpha_spy', 't_spy')} pre08 {cell('pre08', 'alpha_spy', 't_spy')} " f"post08 {cell('post08', 'alpha_spy', 't_spy')} | sh {f.get('sharpe', np.nan):4.2f} " f"mdd {f.get('mdd', np.nan)*100:4.0f}% cagr {f.get('cagr', np.nan)*100:5.1f}%") if "alpha_ew" in f: s += f" | vsEW {f['alpha_ew']*100:5.1f}% t{f['t_ew']:5.2f}" if "corr_mom" in o: b = o["blend50"] s += (f" | corr {o['corr_mom']:4.2f} 50/50 sh {b['full']['blend_sharpe']:.2f} vs {b['full']['mom_sharpe']:.2f}" f" hold {b['holdout']['blend_sharpe']:.2f} vs {b['holdout']['mom_sharpe']:.2f}") return s def ic_table(sig: pd.DataFrame, level: pd.DataFrame, horizons=(3, 6, 12), label: str = "", min_groups: int = 5) -> list[dict]: """Monthly Spearman rank IC of ``sig`` (month-end rows) against the next h-month return of ``level``; Newey-West t with h lags.""" rows = [] me = sig.index lv = level.reindex(me) for h in horizons: fwd = lv.shift(-h) / lv - 1 ics = [] for d in me: x, y = sig.loc[d], fwd.loc[d] ok = x.notna() & y.notna() if ok.sum() >= min_groups: ics.append((d, x[ok].rank().corr(y[ok].rank()))) s = pd.Series(dict(ics)).dropna() row = {"signal": label, "h": h, "n": len(s)} for k, (a, b) in {"full": (None, None), "design": (None, "2016-12-31"), "holdout": ("2017-01-01", None)}.items(): x = s.loc[a:b] if len(x) < 24: continue e = (x - x.mean()).to_numpy() X = np.ones((len(e), 1)) from research.lab.alpha import _nw_se se = _nw_se(X, e, lags=h)[0] row[f"ic_{k}"] = float(x.mean()); row[f"t_{k}"] = float(x.mean() / se) rows.append(row) print(f" IC {label:42s} h{h:2d} n{len(s):4d} full {row.get('ic_full', np.nan):6.3f} t{row.get('t_full', np.nan):5.2f}" f" | des {row.get('ic_design', np.nan):6.3f} t{row.get('t_design', np.nan):5.2f}" f" | hold {row.get('ic_holdout', np.nan):6.3f} t{row.get('t_holdout', np.nan):5.2f}", flush=True) return rows def pos3y(px: pd.DataFrame, win: int = 756) -> pd.DataFrame: lo = px.rolling(win, min_periods=win // 2).min() hi = px.rolling(win, min_periods=win // 2).max() return (px - lo) / (hi - lo) # --------------------------------------------------------------------------- # (a) + (b) + (c) at the ETF level # --------------------------------------------------------------------------- def phase_sector_table(px: pd.DataFrame, me: pd.DatetimeIndex, raw: pd.Series) -> dict: """Does the phase -> sector pattern repeat? Mean monthly sector return in excess of EW9 by CFNAI phase, per period; the favoured-minus-unfavoured spread of the Fidelity map; and the correlation of the 4 x 9 pattern between periods.""" m = px.ffill().reindex(me) fwd = (m.shift(-1) / m - 1).iloc[:-1] ex = fwd.sub(fwd.mean(axis=1), axis=0) phases = {"real-time": asof(phase_cfnai(), me).iloc[:-1], "held month (look-ahead)": pd.Series([raw.get((d + pd.offsets.MonthBegin(1)).normalize()) for d in me[:-1]], index=me[:-1])} periods = {"1999-2007": ("1998-12-01", "2007-12-31"), "2008-2016": ("2008-01-01", "2016-12-31"), "2017-2026": ("2017-01-01", None)} out = {} for pk, ph in phases.items(): mats, spread = {}, {} for per, (a, b) in periods.items(): e, p_ = ex.loc[a:b], ph.loc[a:b] mat = e.groupby(p_).mean().reindex(PHASES) mats[per] = mat sp = {} for phase in PHASES: fav = [x for x in ETF9 if ETF_SECTOR[x] in FAVOURED[phase]] unf = [x for x in ETF9 if ETF_SECTOR[x] in UNFAVOURED[phase]] rows = e[p_ == phase] if len(rows) >= 6 and fav and unf: diff = rows[fav].mean(axis=1) - rows[unf].mean(axis=1) sp[phase] = {"months": len(rows), "fav_minus_unf_ann": float(diff.mean() * 12), "t": float(diff.mean() / diff.std() * np.sqrt(len(diff)))} spread[per] = sp cors = {} keys = list(mats) for i in range(len(keys)): for j in range(i + 1, len(keys)): a_, b_ = mats[keys[i]].stack(), mats[keys[j]].stack() k = a_.index.intersection(b_.index) cors[f"{keys[i]} vs {keys[j]}"] = float(a_[k].corr(b_[k])) out[pk] = {"spread": spread, "pattern_corr": cors, "matrix": {k: (v * 12).round(4).to_dict() for k, v in mats.items()}} print(f" phase -> sector pattern ({pk}): correlation between periods {cors}") for per, sp in spread.items(): print(" " + per + " " + " ".join(f"{ph} {v['fav_minus_unf_ann']*100:5.1f}%/yr t{v['t']:5.2f} ({v['months']}m)" for ph, v in sp.items())) return out def etf_tests(res: dict) -> dict: px = etf_prices().loc["1998-12-22":] me = month_ends(px.index) me = me[me < px.index[-1]] W_ew = pd.DataFrame(1 / 9, index=me, columns=ETF9) ew = etf_book(W_ew, px) res["etf_ew9"] = evaluate("ETF EW 9 sectors (baseline)", ew, "baseline") rets = {"ETF EW9": ew} ph = {"cfnai": asof(phase_cfnai(), me), "indpro": asof(phase_indpro(), me)} for k, s in ph.items(): W = pd.DataFrame(0.0, index=me, columns=ETF9) for d in me: fav = [e for e in ETF9 if ETF_SECTOR[e] in FAVOURED.get(s.loc[d], [])] or ETF9 W.loc[d, fav] = 1 / len(fav) name = f"(a) ETF cycle rotation, {k} phase" rets[name] = etf_book(W, px) res[name] = evaluate(name, rets[name], "a", ew=ew) cnt = s.value_counts() print(" phase months:", cnt.to_dict()) # Upper bound, not a trial: the final-vintage CFNAI phase of the month # about to be held, i.e. perfect knowledge of the current phase. raw = phase_cfnai(raw=True) nxt = pd.Series([raw.get((d + pd.offsets.MonthBegin(1)).normalize()) for d in me], index=me) W = pd.DataFrame(0.0, index=me, columns=ETF9) for d in me: fav = [e for e in ETF9 if ETF_SECTOR[e] in FAVOURED.get(nxt.loc[d], [])] or ETF9 W.loc[d, fav] = 1 / len(fav) name = "bound: ETF rotation with the held month's CFNAI phase (look-ahead)" rets[name] = etf_book(W, px) res[name] = evaluate(name, rets[name], "bound", ew=ew, log=False) res["phase_sector_table"] = phase_sector_table(px, me, raw) c = px.ffill() rf12 = (1 + tbill(px.index).fillna(0) / 252).cumprod() m12_1 = (c.shift(21) / c.shift(252) - 1).reindex(me) r12 = (c / c.shift(252)).reindex(me) rf_12 = (rf12 / rf12.shift(252)).reindex(me) lt = (c.shift(252) / c.shift(1260) - 1).reindex(me) pos = pos3y(c).reindex(me) def topk(sig, k, ascending=False): W = pd.DataFrame(np.nan, index=me, columns=ETF9) for d in me: x = sig.loc[d].dropna() if len(x) < 9: continue pick = x.sort_values(ascending=ascending).index[:k] W.loc[d] = 0.0 W.loc[d, pick] = 1 / k return W variants = { "(b) ETF CS momentum 12-1 top3": topk(m12_1, 3), "(b) ETF TS momentum 12m vs T-bill": (r12.gt(rf_12, axis=0).astype(float) / 9).where(r12.notna()), "(b) ETF long-term reversal 60..13 bottom3": topk(lt, 3, ascending=True), "(c) ETF 3y-range position low3 (contrarian)": topk(pos, 3, ascending=True), "(c) ETF 3y-range position high3": topk(pos, 3), } for name, W in variants.items(): rets[name] = etf_book(W.dropna(how="all"), px) res[name] = evaluate(name, rets[name], name[1], ew=ew) lvl = c.reindex(me) res["ic_etf"] = (ic_table(pos, lvl, label="ETF 3y position") + ic_table(m12_1, lvl, label="ETF mom 12-1") + ic_table(lt, lvl, label="ETF return 60..13 (IC<0 = reversal)")) return rets # --------------------------------------------------------------------------- # constituent level: EW PIT GICS sectors and SIC-2 industries # --------------------------------------------------------------------------- def group_index(p, groups: pd.Series, min_members: int = 3) -> pd.DataFrame: """Daily equal-weight index of each group's PIT members.""" r = p.close.pct_change(fill_method=None).where(p.member.shift(1, fill_value=False)) out = {} for g in groups.dropna().unique(): cols = [c for c in groups.index[groups == g] if c in r.columns] n = r[cols].notna().sum(axis=1) gr = r[cols].mean(axis=1).where(n >= min_members) out[g] = (1 + gr.fillna(0.0)).cumprod().where(n >= min_members) return pd.DataFrame(out) def constituent_tests(p, res: dict) -> dict: sec = p.extra["sector"].reindex(p.close.columns) idx = group_index(p, sec) dates = p.dates[(p.dates >= "2005-01-01")] me = month_ends(dates) me = me[me < dates[-1]] members = p.member sectors = [s for s in idx.columns] def book(chosen): return stock_book(p, sector_weights(chosen, members, sec)) ew = book({d: {s: 1 / len(sectors) for s in sectors} for d in me}) res["gics_ew"] = evaluate("GICS EW sectors, EW members (baseline)", ew, "baseline") rets = {"GICS EW": ew} ph = asof(phase_cfnai(), me) ch = {} for d in me: fav = FAVOURED[ph.loc[d]] if isinstance(ph.loc[d], str) else sectors ch[d] = {s: 1 / len(fav) for s in fav} name = "(a) GICS cycle rotation, cfnai phase" rets[name] = book(ch) res[name] = evaluate(name, rets[name], "a", ew=ew) c = idx.ffill() rf12 = (1 + tbill(c.index).fillna(0) / 252).cumprod() m12_1 = (c.shift(21) / c.shift(252) - 1).reindex(me) r12 = (c / c.shift(252)).reindex(me) rf_12 = (rf12 / rf12.shift(252)).reindex(me) lt = (c.shift(252) / c.shift(1260) - 1).reindex(me) pos = pos3y(c).reindex(me) def topk(sig, k, ascending=False): out = {} for d in me: x = sig.loc[d].dropna() if len(x) < 9: continue out[d] = {s: 1 / k for s in x.sort_values(ascending=ascending).index[:k]} return out ts = {} for d in me: x, f = r12.loc[d], rf_12.loc[d] if x.notna().sum() >= 9: ts[d] = {s: 1 / len(sectors) for s in x.index[x > f]} if not ts[d]: ts[d] = {"__cash__": 0.0} for name, ch in {"(b) GICS CS momentum 12-1 top3": topk(m12_1, 3), "(b) GICS TS momentum 12m vs T-bill": ts, "(b) GICS long-term reversal 60..13 bottom3": topk(lt, 3, ascending=True), "(c) GICS 3y-range position low3": topk(pos, 3, ascending=True)}.items(): rets[name] = book(ch) res[name] = evaluate(name, rets[name], name[1], ew=ew) lvl = c.reindex(me) res["ic_gics"] = (ic_table(pos, lvl, label="GICS 3y position") + ic_table(m12_1, lvl, label="GICS mom 12-1") + ic_table(lt, lvl, label="GICS return 60..13 (IC<0 = reversal)")) # SIC 2-digit industries: more groups for the position signal sic = pd.read_csv(SIC).drop_duplicates("ticker").set_index("ticker")["sic"] sic2 = (pd.to_numeric(sic, errors="coerce") // 100).reindex(p.close.columns) ind = group_index(p, sic2, min_members=4).ffill() ipos = pos3y(ind).reindex(me) ilvl = ind.reindex(me) im = (ind.shift(21) / ind.shift(252) - 1).reindex(me) res["ic_sic2"] = (ic_table(ipos, ilvl, label="SIC2 3y position", min_groups=15) + ic_table(im, ilvl, label="SIC2 mom 12-1 (reference)", min_groups=15)) q = {} for d in me: x = ipos.loc[d].dropna() if len(x) < 15: continue low = x.sort_values().index[: max(3, len(x) // 5)] q[d] = {g: 1 / len(low) for g in low} name = "(c) SIC2 3y-range position bottom quintile" rets[name] = stock_book(p, sector_weights(q, members, sic2)) res[name] = evaluate(name, rets[name], "c", ew=ew) return rets # --------------------------------------------------------------------------- # (c) EDGAR revenue cycle by sector, and the semiconductor cycle # --------------------------------------------------------------------------- def revenue_cycle(p, res: dict) -> dict: """Added after the IC table: the sign (acceleration -> lower 12-month return) was read off 2011-2016 (IC -0.12, t -1.3), so only 2017+ tests the portfolio rule.""" from research.lab import fundamentals as fu F = fu.load(p) g = F["rev_growth"].where(p.member).clip(-0.9, 3.0) sec = p.extra["sector"].reindex(p.close.columns) med = pd.DataFrame({s: g[[c for c in sec.index[sec == s] if c in g.columns]].median(axis=1) for s in sec.dropna().unique()}) idx = group_index(p, sec).ffill() me = month_ends(p.dates[p.dates >= "2011-06-01"]) me = me[me < p.dates[-1]] lvl = idx.reindex(me) level = med.reindex(me) accel = (med - med.shift(126)).reindex(me) res["ic_rev_cycle"] = (ic_table(level, lvl, label="GICS median revenue growth", min_groups=9) + ic_table(accel, lvl, label="GICS revenue growth 6m change", min_groups=9)) ch = {} for d in me: x = accel.loc[d].dropna() if len(x) >= 9: ch[d] = {s: 1 / 3 for s in x.sort_values().index[:3]} name = "(c) GICS 3 sectors with the lowest revenue-growth acceleration" r = stock_book(p, sector_weights(ch, p.member, sec)) ew_ch = {d: {s: 1 / len(idx.columns) for s in idx.columns} for d in ch} ew = stock_book(p, sector_weights(ew_ch, p.member, sec)) res[name] = evaluate(name, r, "c", ew=ew) return {name: r} def semis_cycle(p, res: dict) -> dict: """^SOX vs S&P 500: does the semiconductor cycle position predict the next 3-12 months? Plus two switch rules SMH <-> SPY.""" sox, spx = yahoo("^SOX"), yahoo("^GSPC") j = pd.concat({"sox": sox, "spx": spx}, axis=1).dropna() me = month_ends(j.index) me = me[me < j.index[-1]] rel = j["sox"] / j["spx"] sigs = {"pos3y": pos3y(j[["sox"]])["sox"].reindex(me), "rel_mom12": (rel / rel.shift(252) - 1).reindex(me), "pos3y_rel": pos3y(pd.DataFrame({"r": rel}))["r"].reindex(me)} # SIC 3674 revenue growth acceleration (EDGAR, PIT members, 2011+) from research.lab import fundamentals as fu F = fu.load(p) sic = pd.read_csv(SIC).drop_duplicates("ticker").set_index("ticker")["sic"] semis = [t for t in sic.index[pd.to_numeric(sic, errors="coerce") == 3674] if t in p.close.columns] rg = F["rev_growth"][semis].where(p.member[semis]).clip(-0.9, 3.0).median(axis=1) sigs["rev_growth"] = asof(rg.dropna(), me) sigs["rev_accel6m"] = asof((rg - rg.shift(126)).dropna(), me) lr = np.log(rel.reindex(me)) rows = [] from research.lab.alpha import _nw_se for h in (3, 6, 12): fwd = lr.shift(-h) - lr for k, s in sigs.items(): for span, (a, b) in {"full": (None, None), "to2016": (None, "2016-12-31"), "2017+": ("2017-01-01", None)}.items(): d = pd.concat({"x": s, "y": fwd}, axis=1).loc[a:b].dropna() if len(d) < 36: continue x = (d["x"] - d["x"].mean()) / d["x"].std() X = np.column_stack([np.ones(len(d)), x]) coef, *_ = np.linalg.lstsq(X, d["y"].to_numpy(), rcond=None) se = _nw_se(X, d["y"].to_numpy() - X @ coef, lags=h) rows.append({"signal": k, "h": h, "span": span, "n": len(d), "slope_per_sd": coef[1], "t": coef[1] / se[1], "start": str(d.index[0].date())}) print(f" SOX {k:12s} h{h:2d} {span:6s} n{len(d):4d} slope/sd {coef[1]*100:6.2f}% t {coef[1]/se[1]:5.2f}", flush=True) res["semis_regressions"] = rows px = pd.DataFrame({"SMH": yahoo("SMH"), "SPY": yahoo("SPY")}).dropna() pme = month_ends(px.index) pme = pme[pme < px.index[-1]] rets = {} # The last rule's sign comes from the design-period (<= 2016) regression # above (rel_mom12 -> next 12 months: slope < 0), so only 2017+ tests it. for name, cond in {"(c) semis switch: SMH when SOX/SPX 12m rel mom > 0": sigs["rel_mom12"] > 0, "(c) semis switch: SMH when SOX/SPX 12m rel mom < 0 (sign from <=2016)": sigs["rel_mom12"] < 0, "(c) semis switch: SMH when SOX 3y position < 0.3": sigs["pos3y"] < 0.3, "(c) semis switch: SMH when SIC3674 rev accel > 0": sigs["rev_accel6m"] > 0}.items(): cc = cond.reindex(pme, method="ffill") if name.endswith("rev accel > 0"): cc = cc.where(asof(sigs["rev_accel6m"], pme).notna()) W = pd.DataFrame({"SMH": cc.astype(float), "SPY": 1 - cc.astype(float)}).where(cc.notna()).dropna() rets[name] = etf_book(W, px) res[name] = evaluate(name, rets[name], "c") print(" months in SMH:", int(cc.sum()), "of", int(cc.notna().sum())) rets["SMH buy&hold"] = px["SMH"].pct_change().loc["2000-07-01":] res["SMH buy&hold"] = evaluate("SMH buy & hold (reference)", rets["SMH buy&hold"], "reference", log=False) return rets # --------------------------------------------------------------------------- # (d) the momentum sleeve by macro state, and cycle-conditioned versions # --------------------------------------------------------------------------- def momentum_tests(p, res: dict) -> dict: from research.lab.rotation import factor_ranks from research.lab.us_sweep import _blend, run mom = momentum_sleeve() me = month_ends(mom.index) st = states(me) st_day = st.reindex(mom.index.union(me)).shift(1).ffill().reindex(mom.index) # known at last month-end bq = bench_returns(mom.index, "QQQ") rows = [] for col in st.columns: for val in st[col].dropna().unique(): mask = st_day[col] == val for span, (a, b) in {"design": ("2006-09-27", "2016-12-31"), "holdout": ("2017-01-01", None)}.items(): x = mom.loc[a:b][mask.loc[a:b]] if len(x) < 40: continue q = bq.reindex(x.index) rows.append({"state": col, "value": str(val), "span": span, "days": len(x), "share": len(x) / len(mom.loc[a:b]), "ann_ret": float(x.mean() * 252), "sharpe": float(x.mean() / x.std() * np.sqrt(252)), "qqq_ann_ret": float(q.mean() * 252), "active_ann": float((x - q).mean() * 252)}) S = pd.DataFrame(rows) S.to_csv(os.path.join(RESULTS, "sector_cycle_states.csv"), index=False) print(S.to_string(float_format=lambda v: f"{v:.2f}"), flush=True) res["states"] = rows # cash switch in the design-worst state (>= 10% of design days) d = S[(S["span"] == "design") & (S["share"] >= 0.10) & (S["share"] <= 0.9)].sort_values("sharpe") worst = d.iloc[0] print(f"design-worst state: {worst['state']}={worst['value']} (Sharpe {worst['sharpe']:.2f}, share {worst['share']:.2f})") res["cash_switch_state"] = {"state": worst["state"], "value": worst["value"], "design_sharpe": float(worst["sharpe"])} off = (st_day[worst["state"]].astype(str) == worst["value"]).astype(float) rf = tbill(mom.index).fillna(0) / 252 switch_cost = off.diff().abs().fillna(0) * 0.001 # 10 bp to sell all or buy all rets = {} name = f"(d) momentum, cash when {worst['state']}={worst['value']}" rets[name] = mom * (1 - off) + rf * off - switch_cost res[name] = evaluate(name, rets[name], "d") res["momentum sleeve"] = evaluate("momentum sleeve (overnight_recovery_top10)", mom, "reference", log=False) # picks restricted by the CFNAI phase pr = factor_ranks(p, "us", ["overnight_mom", "rec_126"]) score = _blend(pr["overnight_mom"], pr["rec_126"]).where(p.member) ph = asof(phase_cfnai(), p.dates) sec = p.extra["sector"].reindex(p.close.columns) fav = pd.DataFrame(False, index=p.dates, columns=p.close.columns) unf = pd.DataFrame(True, index=p.dates, columns=p.close.columns) for phase in PHASES: rowsel = (ph == phase).to_numpy() fav.loc[rowsel] = np.repeat(sec.isin(FAVOURED[phase]).to_numpy()[None, :], rowsel.sum(), axis=0) unf.loc[rowsel] = np.repeat((~sec.isin(UNFAVOURED[phase])).to_numpy()[None, :], rowsel.sum(), axis=0) base = run(p, score, [LONG], top_n=10, buffer=2.0, every=10)["_rets"] res["momentum rerun"] = evaluate("momentum sleeve (rerun, same engine)", base, "reference", log=False) for name, m in {"(d) momentum, favoured sectors of CFNAI phase only": fav, "(d) momentum, unfavoured sectors of CFNAI phase removed": unf}.items(): rets[name] = run(p, score.where(m), [LONG], top_n=10, buffer=2.0, every=10)["_rets"] res[name] = evaluate(name, rets[name], "d") return rets def main() -> None: if len(sys.argv) > 1 and sys.argv[1] == "fetch": fetch() return from research.lab.factor_scan import load only = sys.argv[1:] or ["etf", "constituent", "revenue", "semis", "momentum"] res, rets = {}, {} p = None if set(only) - {"etf"}: p, _ = load("us") if "etf" in only: print("== ETF level (a)(b)(c)", flush=True) rets.update(etf_tests(res)) if "constituent" in only: print("== constituent level (a)(b)(c)", flush=True) rets.update(constituent_tests(p, res)) if "revenue" in only: print("== EDGAR revenue cycle (c)", flush=True) rets.update(revenue_cycle(p, res)) if "semis" in only: print("== semiconductor cycle (c)", flush=True) rets.update(semis_cycle(p, res)) if "momentum" in only: print("== momentum sleeve conditioning (d)", flush=True) rets.update(momentum_tests(p, res)) # one results file per kind; a partial run updates only its own keys jpath, rpath = os.path.join(RESULTS, "sector_cycle.json"), os.path.join(RESULTS, "sector_cycle_returns.pkl") old = json.load(open(jpath)) if os.path.exists(jpath) else {} with open(jpath, "w") as fh: json.dump({**old, **res}, fh, indent=1, default=float) R = pd.read_pickle(rpath) if os.path.exists(rpath) else pd.DataFrame() new = pd.DataFrame(rets) R = pd.concat([R.drop(columns=[c for c in new.columns if c in R.columns]), new], axis=1) R.to_pickle(rpath) flat = [] for t in TRIALS: row = {"name": t["name"], "test": t["test"], "start": t["start"]} for k in SPANS: for m, v in t.get(k, {}).items(): row[f"{k}_{m}"] = v row["corr_mom"] = t.get("corr_mom") flat.append(row) path = os.path.join(RESULTS, "sector_cycle_trials.csv") pd.DataFrame(flat).to_csv(path, mode="a", header=not os.path.exists(path), index=False) print(f"{len(TRIALS)} trials logged to {path}") if __name__ == "__main__": main()