"""Timestamped Yahoo analyst snapshots and a forward-only momentum overlay. uv run python -m research.lab.analyst_estimates collect --candidates data/agent/us/candidates_.json uv run python -m research.lab.analyst_estimates evaluate The vendor's 30/90-day EPS trend is known only when collected. It never creates observations on those earlier dates. Price targets are descriptive; the research overlay uses EPS revisions within the quant's top 50 names. """ from __future__ import annotations import argparse import hashlib import json import math import time from datetime import datetime, timezone from pathlib import Path import numpy as np import pandas as pd from strategies.agent_review import session_deadline from strategies.robust_momentum import RobustMomentumStrategy ROOT = Path(__file__).resolve().parents[2] SNAPSHOTS = ROOT / "data/research/analyst_estimates" PROVIDER = "yahoo_finance" MAX_AGE_DAYS = 7 MIN_ANALYSTS = 5 EPS_FLOOR = 0.25 POOL = 50 MIN_COVERAGE = 0.70 REVISION_WEIGHT = 0.20 FEATURES = ("eps_revision_30d", "eps_revision_90d", "revision_balance_30d") def _utc(value) -> pd.Timestamp: ts = pd.Timestamp(value) if pd.isna(ts) or ts.tzinfo is None: raise ValueError("Snapshot timestamps must be timezone-aware") return ts.tz_convert("UTC") def _clean(value): if isinstance(value, dict): return {str(k): _clean(v) for k, v in value.items()} if isinstance(value, (list, tuple)): return [_clean(v) for v in value] if value is None or pd.isna(value): return None if isinstance(value, np.generic): return value.item() return value def collect(tickers: list[str], directory: Path = SNAPSHOTS) -> Path: import yfinance as yf from research.lab.fetch_us import _proxy_env _proxy_env() rows = {} for symbol in dict.fromkeys(tickers): ticker = yf.Ticker(symbol) row = {"source_url": f"https://finance.yahoo.com/quote/{symbol}/analysis/", "errors": []} for field in ("earnings_estimate", "revenue_estimate", "eps_trend", "eps_revisions", "analyst_price_targets"): try: value = getattr(ticker, field) row[field] = _clean(value.to_dict(orient="index") if isinstance(value, pd.DataFrame) else value) if not row[field]: row["errors"].append(f"{field}: unavailable") except Exception as exc: # Preserve missing endpoints; never turn a failure into a zero signal. row[field] = {} row["errors"].append(f"{field}: {type(exc).__name__}") # yfinance's public estimate tables drop endDate. Preserve the fetched # period identity to make fiscal rollovers and stale metadata visible. trend = ticker._analysis._earnings_trend or [] row["period_ends"] = {r["period"]: r.get("endDate") for r in trend} row["observed_at"] = datetime.now(timezone.utc).isoformat() rows[symbol] = row print(f"{symbol}: {'available' if row['eps_trend'] else 'missing EPS trend'}", flush=True) time.sleep(0.2) now = datetime.now(timezone.utc) snapshot = {"schema": 1, "provider": PROVIDER, "created_at": now.isoformat(), "yfinance_version": yf.__version__, "rows": rows} directory = Path(directory) directory.mkdir(parents=True, exist_ok=True) path = directory / f"snapshot_{now:%Y%m%dT%H%M%S%fZ}.json" with path.open("x") as fh: json.dump(snapshot, fh, indent=1, allow_nan=False) print(f"Saved {len(rows)} immutable observations -> {path}", flush=True) return path def load_snapshots(directory: Path = SNAPSHOTS) -> list[dict]: snapshots = [] for path in sorted(Path(directory).glob("snapshot_*.json")): d = json.loads(path.read_text()) if d.get("schema") != 1 or d.get("provider") != PROVIDER: raise ValueError(f"Unsupported analyst snapshot: {path}") created = _utc(d["created_at"]) for row in d["rows"].values(): if _utc(row["observed_at"]) > created: raise ValueError(f"Observation newer than its snapshot: {path}") snapshots.append(d) return sorted(snapshots, key=lambda d: _utc(d["created_at"])) def _number(value) -> float | None: if isinstance(value, bool): return None try: v = float(value) return v if math.isfinite(v) else None except (ValueError, TypeError): return None def row_features(row: dict) -> dict: period = "+1y" trend = row.get("eps_trend", {}).get(period, {}) estimate = row.get("earnings_estimate", {}).get(period, {}) revisions = row.get("eps_revisions", {}).get(period, {}) result = {"period": period, "period_end": row.get("period_ends", {}).get(period), "observed_at": row["observed_at"], "source_url": row["source_url"], "eps": _number(trend.get("current")), "analysts": _number(estimate.get("numberOfAnalysts")), **{k: None for k in FEATURES}, "eligible": False, "quality_flags": []} end = pd.to_datetime(result["period_end"], errors="coerce") observed = _utc(row["observed_at"]).tz_localize(None).normalize() if pd.isna(end) or end <= observed: result["quality_flags"].append("missing_or_expired_fiscal_period") current_end = pd.to_datetime(row.get("period_ends", {}).get("0y"), errors="coerce") if pd.isna(current_end) or current_end <= observed: result["quality_flags"].append("stale_current_year_metadata") if not pd.isna(end) and not pd.isna(current_end) and end <= current_end: result["quality_flags"].append("inconsistent_fiscal_period_order") if (result["analysts"] or 0) < MIN_ANALYSTS: result["quality_flags"].append("insufficient_analyst_coverage") current = result["eps"] for days in (30, 90): old = _number(trend.get(f"{days}daysAgo")) if current is not None and current > EPS_FLOOR and old is not None and old > EPS_FLOOR: result[f"eps_revision_{days}d"] = current / old - 1.0 up, down = (_number(revisions.get(k)) for k in ("upLast30days", "downLast30days")) if up is not None and down is not None and min(up, down) >= 0: result["revision_balance_30d"] = (up - down) / (up + down) if up + down else 0.0 if any(result[k] is None for k in FEATURES): result["quality_flags"].append("missing_revisions_or_small_nonpositive_eps") if current is not None and _number(estimate.get("avg")) is not None: if not np.isclose(current, float(estimate["avg"]), rtol=0.01, atol=0.01): result["quality_flags"].append("inconsistent_consensus_eps") if current is not None and current > EPS_FLOOR: lo, hi = (_number(estimate.get(k)) for k in ("low", "high")) result["eps_range_over_mean"] = (hi - lo) / current if lo is not None and hi is not None else None targets = row.get("analyst_price_targets", {}) price, target = _number(targets.get("current")), _number(targets.get("median")) result["target_median_upside"] = target / price - 1 if price and price > 0 and target and target > 0 else None result["price_targets"] = targets # Old 0y metadata is a data-quality gate, even when +1y still lies ahead. result["eligible"] = not result["quality_flags"] return result def candidate_context(tickers: list[str], cutoff, snapshots: list[dict] | None = None) -> dict[str, dict]: cutoff = _utc(cutoff) out = {} ordered = load_snapshots() if snapshots is None else sorted(snapshots, key=lambda d: _utc(d["created_at"])) for d in ordered: created = _utc(d["created_at"]) if not cutoff - pd.Timedelta(days=MAX_AGE_DAYS) <= created <= cutoff: continue for ticker in tickers: if ticker in d["rows"]: row = d["rows"][ticker] if cutoff - pd.Timedelta(days=MAX_AGE_DAYS) <= _utc(row["observed_at"]) <= cutoff: out[ticker] = {"provider": d["provider"], "snapshot_created_at": d["created_at"], **row_features(row)} return out def feature_history(dates, columns, snapshots: list[dict]) -> dict[str, pd.DataFrame]: frames = {k: pd.DataFrame(np.nan, index=dates, columns=columns) for k in FEATURES} for day in dates: cutoff = session_deadline(day) context = candidate_context(list(columns), cutoff, snapshots) for ticker, row in context.items(): if row["eligible"]: for k in FEATURES: frames[k].at[day, ticker] = row[k] return frames def blend_revisions(base: pd.DataFrame, features: dict[str, pd.DataFrame]) -> tuple[pd.DataFrame, pd.Series]: output = base.copy() active = pd.Series(False, index=base.index) for day, row in base.iterrows(): candidates = row.dropna().sort_values(ascending=False, kind="stable").head(POOL).index f = pd.DataFrame({k: features[k].loc[day].reindex(candidates) for k in FEATURES}) valid = f.notna().all(axis=1) if len(f) < 10 or valid.sum() < math.ceil(len(f) * MIN_COVERAGE): continue # A tie carries no directional information; missing names stay neutral. rank = (f.loc[valid].rank(method="average") - 1) / max(int(valid.sum()) - 1, 1) score = 0.50 * rank[FEATURES[0]] + 0.25 * rank[FEATURES[1]] + 0.25 * rank[FEATURES[2]] quant = pd.Series(1 - np.arange(len(f)) / max(len(f) - 1, 1), index=candidates) output.loc[day, row.notna()] = -2.0 output.loc[day, candidates] = quant + REVISION_WEIGHT * (2 * score.reindex(candidates).fillna(0.5) - 1) active.at[day] = True return output, active class AnalystRevisionStrategy(RobustMomentumStrategy): """Research only: reuse the base strategy's timing, turnover buffer and sizing.""" def __init__(self, snapshots: list[dict], **kwargs): super().__init__("overnight_recovery", **kwargs) self.snapshots = snapshots self._quant_score = self._score self._score = self._revision_score self.active = pd.Series(dtype=bool) def _revision_score(self, panel): base = self._quant_score(panel) eligible = panel["close"].notna() if "eligible" in panel: eligible &= panel["eligible"].reindex_like(base).fillna(False).astype(bool) base = base.where(eligible) features = feature_history(base.index, base.columns, self.snapshots) score, self.active = blend_revisions(base, features) return score def evaluate(directory: Path = SNAPSHOTS, destination: Path | None = None) -> dict: from backtest_runner import ENGINE_VERSION, load_stock_panels, prepare_strategy, replay_prepared from research.lab.trials import append_trials import metrics snapshots = load_snapshots(directory) stamp = datetime.now(timezone.utc) destination = destination or ROOT / f"data/research/results/analyst_revisions_{stamp:%Y%m%dT%H%M%S%fZ}" destination.mkdir(parents=True, exist_ok=False) protocol = { "status": "approved_local_forward_research", "created_at": stamp.isoformat(), "claim": "EPS revisions add information beyond overnight/recovery momentum", "competing_hypothesis": "Analysts follow the same price trend; the overlay adds turnover without alpha", "prediction": "Net CAGR and Sharpe rise on the same sessions after collection; gains survive higher costs", "engine": ENGINE_VERSION, "baseline": "overnight_recovery_top10", "pool": POOL, "revision_weight": REVISION_WEIGHT, "coverage_gate": MIN_COVERAGE, "features": dict(zip(FEATURES, (0.50, 0.25, 0.25))), "target_prices": "descriptive only; not a return/probability or a selection factor", "controls": "same PIT membership, warmup, close fills, top10, rebalance10, buffer2, whole shares", "costs_per_side": [0.001, 0.0025], "capital": 10000, "commission": "max($2, $1.88 + $0.009/share)", "minimum_evidence": "252 active signal sessions, 20 distinct collection dates, active on >=70% of evaluated sessions", "decision_rule": "Research candidate only: CAGR improves and Sharpe +0.10 at both costs; MaxDD deterioration <=2pp", "promotion_boundary": "No live promotion; historical searches already examined are development, not independent OOS", "data_boundary": "Vendor EPS trend is usable only after created_at; stale fiscal metadata and loss/near-zero EPS excluded", "metrics": "calendar-time CAGR, daily-return Sharpe (rf=0), MaxDD, orders, fees, paired net returns", "ablations": "base vs fixed EPS-only overlay; 10bp vs 25bp; no parameter search", "expected_artifacts": ["protocol.json", "results.json", "returns.csv when enough observations exist"], "snapshots": {str(p): hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(Path(directory).glob("snapshot_*.json"))}, "code_hashes": {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in (Path(__file__), ROOT / "strategies/robust_momentum.py", ROOT / "execution.py", ROOT / "signals.py", ROOT / "backtest_runner.py", ROOT / "metrics.py")}, "data_hashes": {str(p): hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT / "data/us.csv", ROOT / "data/us_open.csv", ROOT / "data/us_volume.csv", ROOT / "data/sp500_history.json") if p.exists()}, } (destination / "protocol.json").write_text(json.dumps(protocol, indent=2)) result = {"verdict": "NEEDS EVIDENCE", "snapshot_count": len(snapshots), "comparisons": []} if snapshots: stock = load_stock_panels("us", [protocol["baseline"]], None, refresh=False) prices = stock["panels"]["close"] first = _utc(snapshots[0]["created_at"]).tz_convert("America/New_York").date() days = prices.index[prices.index.date >= first] result["priced_sessions_after_collection"] = len(days) if len(days) >= 2: strategy = AnalystRevisionStrategy(snapshots) args = dict(member=stock["member"], start=str(days[0].date()), end=str(days[-1].date())) plan = prepare_strategy(protocol["baseline"], "us", stock["panels"], stock["stocks"], strategy=strategy, **args) active = strategy.active.loc[days[0]:days[-1]] result["active_signal_sessions"] = int(active.sum()) result["snapshot_collection_dates"] = len({_utc(d["created_at"]).date() for d in snapshots if _utc(d["created_at"]) <= session_deadline(days[-1])}) if active.any(): start = active.index[active][0] + pd.offsets.BDay(1) # Shifted signals trade one session after the collection cutoff. live_days = plan["prices"].index[plan["prices"].index >= start] result["evaluated_sessions"] = len(live_days) if len(live_days) >= 2: args["start"] = str(live_days[0].date()) plan = prepare_strategy(protocol["baseline"], "us", stock["panels"], stock["stocks"], strategy=strategy, **args) base = prepare_strategy(protocol["baseline"], "us", stock["panels"], stock["stocks"], **args) enough = (int(active.sum()) >= 252 and result["snapshot_collection_dates"] >= 20 and float(active.mean()) >= MIN_COVERAGE) returns = {} for cost in protocol["costs_per_side"]: pair = {} for name, p in (("base", base), ("eps_revisions", plan)): run = replay_prepared(p, 10000, cost, 2.0, fee_base=1.88, fee_per_share=0.009) r = run["equity"].pct_change().dropna() returns[f"{name}_{cost}"] = r pair[name] = {"orders": len(run["trade_log"]), "fees": sum(t["commission"] for t in run["trade_log"]), "total_return": float(run["equity"].iloc[-1] / run["equity"].iloc[0] - 1)} if enough: pair[name].update(metrics.raw_summary(run["equity"])) result["comparisons"].append({"cost": cost, **pair}) pd.DataFrame(returns).to_csv(destination / "returns.csv") if enough: passed = all(c["eps_revisions"]["annualizedReturn"] > c["base"]["annualizedReturn"] and c["eps_revisions"]["sharpeRatio"] >= c["base"]["sharpeRatio"] + 0.10 and c["eps_revisions"]["maxDrawdown"] >= c["base"]["maxDrawdown"] - 0.02 for c in result["comparisons"]) result["verdict"] = "FORWARD CANDIDATE" if passed else "RETAIN BASELINE" append_trials(destination / "trials.jsonl", result["comparisons"], protocol) result["boundary"] = "Insufficient forward evidence never receives annualized performance or a promotion verdict." (destination / "results.json").write_text(json.dumps(result, indent=2, allow_nan=False)) print(json.dumps(result, indent=2), flush=True) print(f"Saved -> {destination}", flush=True) return result def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="action", required=True) p = sub.add_parser("collect") g = p.add_mutually_exclusive_group(required=True) g.add_argument("--candidates", type=Path) g.add_argument("--tickers", nargs="+") p.add_argument("--directory", type=Path, default=SNAPSHOTS) p = sub.add_parser("evaluate") p.add_argument("--directory", type=Path, default=SNAPSHOTS) p.add_argument("--output", type=Path) args = ap.parse_args() if args.action == "collect": tickers = args.tickers if args.tickers else [r["ticker"] for r in json.loads(args.candidates.read_text())["candidates"]] collect(tickers, args.directory) else: evaluate(args.directory, args.output) if __name__ == "__main__": main()