Gahow Wang
0e1e0a202c
collect archives Yahoo's earnings/revenue estimates, EPS trend, EPS revisions and price targets as immutable snapshots; a snapshot is usable only after its created_at, so the vendor's 30/90-day history never reaches earlier bars. evaluate replays overnight_recovery_top10 with an EPS-revision re-rank of the quant top 50 on the same ledger and costs and returns NEEDS EVIDENCE until 252 active sessions from 20 collection dates exist. The agent's candidate pack now carries each name's analyst context. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
349 lines
18 KiB
Python
349 lines
18 KiB
Python
"""Timestamped Yahoo analyst snapshots and a forward-only momentum overlay.
|
|
|
|
uv run python -m research.lab.analyst_estimates collect --candidates data/agent/us/candidates_<as_of>.json
|
|
uv run python -m research.lab.analyst_estimates evaluate
|
|
|
|
The vendor's 30/90-day EPS trend is known only when collected. It never
|
|
creates observations on those earlier dates. Price targets are descriptive;
|
|
the research overlay uses EPS revisions within the quant's top 50 names.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
from strategies.agent_review import session_deadline
|
|
from strategies.robust_momentum import RobustMomentumStrategy
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SNAPSHOTS = ROOT / "data/research/analyst_estimates"
|
|
PROVIDER = "yahoo_finance"
|
|
MAX_AGE_DAYS = 7
|
|
MIN_ANALYSTS = 5
|
|
EPS_FLOOR = 0.25
|
|
POOL = 50
|
|
MIN_COVERAGE = 0.70
|
|
REVISION_WEIGHT = 0.20
|
|
FEATURES = ("eps_revision_30d", "eps_revision_90d", "revision_balance_30d")
|
|
|
|
|
|
def _utc(value) -> pd.Timestamp:
|
|
ts = pd.Timestamp(value)
|
|
if pd.isna(ts) or ts.tzinfo is None:
|
|
raise ValueError("Snapshot timestamps must be timezone-aware")
|
|
return ts.tz_convert("UTC")
|
|
|
|
|
|
def _clean(value):
|
|
if isinstance(value, dict):
|
|
return {str(k): _clean(v) for k, v in value.items()}
|
|
if isinstance(value, (list, tuple)):
|
|
return [_clean(v) for v in value]
|
|
if value is None or pd.isna(value):
|
|
return None
|
|
if isinstance(value, np.generic):
|
|
return value.item()
|
|
return value
|
|
|
|
|
|
def collect(tickers: list[str], directory: Path = SNAPSHOTS) -> Path:
|
|
import yfinance as yf
|
|
from research.lab.fetch_us import _proxy_env
|
|
|
|
_proxy_env()
|
|
rows = {}
|
|
for symbol in dict.fromkeys(tickers):
|
|
ticker = yf.Ticker(symbol)
|
|
row = {"source_url": f"https://finance.yahoo.com/quote/{symbol}/analysis/", "errors": []}
|
|
for field in ("earnings_estimate", "revenue_estimate", "eps_trend", "eps_revisions",
|
|
"analyst_price_targets"):
|
|
try:
|
|
value = getattr(ticker, field)
|
|
row[field] = _clean(value.to_dict(orient="index") if isinstance(value, pd.DataFrame) else value)
|
|
if not row[field]:
|
|
row["errors"].append(f"{field}: unavailable")
|
|
except Exception as exc: # Preserve missing endpoints; never turn a failure into a zero signal.
|
|
row[field] = {}
|
|
row["errors"].append(f"{field}: {type(exc).__name__}")
|
|
# yfinance's public estimate tables drop endDate. Preserve the fetched
|
|
# period identity to make fiscal rollovers and stale metadata visible.
|
|
trend = ticker._analysis._earnings_trend or []
|
|
row["period_ends"] = {r["period"]: r.get("endDate") for r in trend}
|
|
row["observed_at"] = datetime.now(timezone.utc).isoformat()
|
|
rows[symbol] = row
|
|
print(f"{symbol}: {'available' if row['eps_trend'] else 'missing EPS trend'}", flush=True)
|
|
time.sleep(0.2)
|
|
now = datetime.now(timezone.utc)
|
|
snapshot = {"schema": 1, "provider": PROVIDER, "created_at": now.isoformat(),
|
|
"yfinance_version": yf.__version__, "rows": rows}
|
|
directory = Path(directory)
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
path = directory / f"snapshot_{now:%Y%m%dT%H%M%S%fZ}.json"
|
|
with path.open("x") as fh:
|
|
json.dump(snapshot, fh, indent=1, allow_nan=False)
|
|
print(f"Saved {len(rows)} immutable observations -> {path}", flush=True)
|
|
return path
|
|
|
|
|
|
def load_snapshots(directory: Path = SNAPSHOTS) -> list[dict]:
|
|
snapshots = []
|
|
for path in sorted(Path(directory).glob("snapshot_*.json")):
|
|
d = json.loads(path.read_text())
|
|
if d.get("schema") != 1 or d.get("provider") != PROVIDER:
|
|
raise ValueError(f"Unsupported analyst snapshot: {path}")
|
|
created = _utc(d["created_at"])
|
|
for row in d["rows"].values():
|
|
if _utc(row["observed_at"]) > created:
|
|
raise ValueError(f"Observation newer than its snapshot: {path}")
|
|
snapshots.append(d)
|
|
return sorted(snapshots, key=lambda d: _utc(d["created_at"]))
|
|
|
|
|
|
def _number(value) -> float | None:
|
|
if isinstance(value, bool):
|
|
return None
|
|
try:
|
|
v = float(value)
|
|
return v if math.isfinite(v) else None
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def row_features(row: dict) -> dict:
|
|
period = "+1y"
|
|
trend = row.get("eps_trend", {}).get(period, {})
|
|
estimate = row.get("earnings_estimate", {}).get(period, {})
|
|
revisions = row.get("eps_revisions", {}).get(period, {})
|
|
result = {"period": period, "period_end": row.get("period_ends", {}).get(period),
|
|
"observed_at": row["observed_at"], "source_url": row["source_url"],
|
|
"eps": _number(trend.get("current")),
|
|
"analysts": _number(estimate.get("numberOfAnalysts")),
|
|
**{k: None for k in FEATURES}, "eligible": False, "quality_flags": []}
|
|
end = pd.to_datetime(result["period_end"], errors="coerce")
|
|
observed = _utc(row["observed_at"]).tz_localize(None).normalize()
|
|
if pd.isna(end) or end <= observed:
|
|
result["quality_flags"].append("missing_or_expired_fiscal_period")
|
|
current_end = pd.to_datetime(row.get("period_ends", {}).get("0y"), errors="coerce")
|
|
if pd.isna(current_end) or current_end <= observed:
|
|
result["quality_flags"].append("stale_current_year_metadata")
|
|
if not pd.isna(end) and not pd.isna(current_end) and end <= current_end:
|
|
result["quality_flags"].append("inconsistent_fiscal_period_order")
|
|
if (result["analysts"] or 0) < MIN_ANALYSTS:
|
|
result["quality_flags"].append("insufficient_analyst_coverage")
|
|
current = result["eps"]
|
|
for days in (30, 90):
|
|
old = _number(trend.get(f"{days}daysAgo"))
|
|
if current is not None and current > EPS_FLOOR and old is not None and old > EPS_FLOOR:
|
|
result[f"eps_revision_{days}d"] = current / old - 1.0
|
|
up, down = (_number(revisions.get(k)) for k in ("upLast30days", "downLast30days"))
|
|
if up is not None and down is not None and min(up, down) >= 0:
|
|
result["revision_balance_30d"] = (up - down) / (up + down) if up + down else 0.0
|
|
if any(result[k] is None for k in FEATURES):
|
|
result["quality_flags"].append("missing_revisions_or_small_nonpositive_eps")
|
|
if current is not None and _number(estimate.get("avg")) is not None:
|
|
if not np.isclose(current, float(estimate["avg"]), rtol=0.01, atol=0.01):
|
|
result["quality_flags"].append("inconsistent_consensus_eps")
|
|
if current is not None and current > EPS_FLOOR:
|
|
lo, hi = (_number(estimate.get(k)) for k in ("low", "high"))
|
|
result["eps_range_over_mean"] = (hi - lo) / current if lo is not None and hi is not None else None
|
|
targets = row.get("analyst_price_targets", {})
|
|
price, target = _number(targets.get("current")), _number(targets.get("median"))
|
|
result["target_median_upside"] = target / price - 1 if price and price > 0 and target and target > 0 else None
|
|
result["price_targets"] = targets
|
|
# Old 0y metadata is a data-quality gate, even when +1y still lies ahead.
|
|
result["eligible"] = not result["quality_flags"]
|
|
return result
|
|
|
|
|
|
def candidate_context(tickers: list[str], cutoff, snapshots: list[dict] | None = None) -> dict[str, dict]:
|
|
cutoff = _utc(cutoff)
|
|
out = {}
|
|
ordered = load_snapshots() if snapshots is None else sorted(snapshots, key=lambda d: _utc(d["created_at"]))
|
|
for d in ordered:
|
|
created = _utc(d["created_at"])
|
|
if not cutoff - pd.Timedelta(days=MAX_AGE_DAYS) <= created <= cutoff:
|
|
continue
|
|
for ticker in tickers:
|
|
if ticker in d["rows"]:
|
|
row = d["rows"][ticker]
|
|
if cutoff - pd.Timedelta(days=MAX_AGE_DAYS) <= _utc(row["observed_at"]) <= cutoff:
|
|
out[ticker] = {"provider": d["provider"], "snapshot_created_at": d["created_at"],
|
|
**row_features(row)}
|
|
return out
|
|
|
|
|
|
def feature_history(dates, columns, snapshots: list[dict]) -> dict[str, pd.DataFrame]:
|
|
frames = {k: pd.DataFrame(np.nan, index=dates, columns=columns) for k in FEATURES}
|
|
for day in dates:
|
|
cutoff = session_deadline(day)
|
|
context = candidate_context(list(columns), cutoff, snapshots)
|
|
for ticker, row in context.items():
|
|
if row["eligible"]:
|
|
for k in FEATURES:
|
|
frames[k].at[day, ticker] = row[k]
|
|
return frames
|
|
|
|
|
|
def blend_revisions(base: pd.DataFrame, features: dict[str, pd.DataFrame]) -> tuple[pd.DataFrame, pd.Series]:
|
|
output = base.copy()
|
|
active = pd.Series(False, index=base.index)
|
|
for day, row in base.iterrows():
|
|
candidates = row.dropna().sort_values(ascending=False, kind="stable").head(POOL).index
|
|
f = pd.DataFrame({k: features[k].loc[day].reindex(candidates) for k in FEATURES})
|
|
valid = f.notna().all(axis=1)
|
|
if len(f) < 10 or valid.sum() < math.ceil(len(f) * MIN_COVERAGE):
|
|
continue
|
|
# A tie carries no directional information; missing names stay neutral.
|
|
rank = (f.loc[valid].rank(method="average") - 1) / max(int(valid.sum()) - 1, 1)
|
|
score = 0.50 * rank[FEATURES[0]] + 0.25 * rank[FEATURES[1]] + 0.25 * rank[FEATURES[2]]
|
|
quant = pd.Series(1 - np.arange(len(f)) / max(len(f) - 1, 1), index=candidates)
|
|
output.loc[day, row.notna()] = -2.0
|
|
output.loc[day, candidates] = quant + REVISION_WEIGHT * (2 * score.reindex(candidates).fillna(0.5) - 1)
|
|
active.at[day] = True
|
|
return output, active
|
|
|
|
|
|
class AnalystRevisionStrategy(RobustMomentumStrategy):
|
|
"""Research only: reuse the base strategy's timing, turnover buffer and sizing."""
|
|
def __init__(self, snapshots: list[dict], **kwargs):
|
|
super().__init__("overnight_recovery", **kwargs)
|
|
self.snapshots = snapshots
|
|
self._quant_score = self._score
|
|
self._score = self._revision_score
|
|
self.active = pd.Series(dtype=bool)
|
|
|
|
def _revision_score(self, panel):
|
|
base = self._quant_score(panel)
|
|
eligible = panel["close"].notna()
|
|
if "eligible" in panel:
|
|
eligible &= panel["eligible"].reindex_like(base).fillna(False).astype(bool)
|
|
base = base.where(eligible)
|
|
features = feature_history(base.index, base.columns, self.snapshots)
|
|
score, self.active = blend_revisions(base, features)
|
|
return score
|
|
|
|
|
|
def evaluate(directory: Path = SNAPSHOTS, destination: Path | None = None) -> dict:
|
|
from backtest_runner import ENGINE_VERSION, load_stock_panels, prepare_strategy, replay_prepared
|
|
from research.lab.trials import append_trials
|
|
import metrics
|
|
|
|
snapshots = load_snapshots(directory)
|
|
stamp = datetime.now(timezone.utc)
|
|
destination = destination or ROOT / f"data/research/results/analyst_revisions_{stamp:%Y%m%dT%H%M%S%fZ}"
|
|
destination.mkdir(parents=True, exist_ok=False)
|
|
protocol = {
|
|
"status": "approved_local_forward_research", "created_at": stamp.isoformat(),
|
|
"claim": "EPS revisions add information beyond overnight/recovery momentum",
|
|
"competing_hypothesis": "Analysts follow the same price trend; the overlay adds turnover without alpha",
|
|
"prediction": "Net CAGR and Sharpe rise on the same sessions after collection; gains survive higher costs",
|
|
"engine": ENGINE_VERSION, "baseline": "overnight_recovery_top10",
|
|
"pool": POOL, "revision_weight": REVISION_WEIGHT, "coverage_gate": MIN_COVERAGE,
|
|
"features": dict(zip(FEATURES, (0.50, 0.25, 0.25))),
|
|
"target_prices": "descriptive only; not a return/probability or a selection factor",
|
|
"controls": "same PIT membership, warmup, close fills, top10, rebalance10, buffer2, whole shares",
|
|
"costs_per_side": [0.001, 0.0025], "capital": 10000,
|
|
"commission": "max($2, $1.88 + $0.009/share)",
|
|
"minimum_evidence": "252 active signal sessions, 20 distinct collection dates, active on >=70% of evaluated sessions",
|
|
"decision_rule": "Research candidate only: CAGR improves and Sharpe +0.10 at both costs; MaxDD deterioration <=2pp",
|
|
"promotion_boundary": "No live promotion; historical searches already examined are development, not independent OOS",
|
|
"data_boundary": "Vendor EPS trend is usable only after created_at; stale fiscal metadata and loss/near-zero EPS excluded",
|
|
"metrics": "calendar-time CAGR, daily-return Sharpe (rf=0), MaxDD, orders, fees, paired net returns",
|
|
"ablations": "base vs fixed EPS-only overlay; 10bp vs 25bp; no parameter search",
|
|
"expected_artifacts": ["protocol.json", "results.json", "returns.csv when enough observations exist"],
|
|
"snapshots": {str(p): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in sorted(Path(directory).glob("snapshot_*.json"))},
|
|
"code_hashes": {p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in (Path(__file__), ROOT / "strategies/robust_momentum.py", ROOT / "execution.py",
|
|
ROOT / "signals.py", ROOT / "backtest_runner.py", ROOT / "metrics.py")},
|
|
"data_hashes": {str(p): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in (ROOT / "data/us.csv", ROOT / "data/us_open.csv", ROOT / "data/us_volume.csv",
|
|
ROOT / "data/sp500_history.json") if p.exists()},
|
|
}
|
|
(destination / "protocol.json").write_text(json.dumps(protocol, indent=2))
|
|
result = {"verdict": "NEEDS EVIDENCE", "snapshot_count": len(snapshots), "comparisons": []}
|
|
if snapshots:
|
|
stock = load_stock_panels("us", [protocol["baseline"]], None, refresh=False)
|
|
prices = stock["panels"]["close"]
|
|
first = _utc(snapshots[0]["created_at"]).tz_convert("America/New_York").date()
|
|
days = prices.index[prices.index.date >= first]
|
|
result["priced_sessions_after_collection"] = len(days)
|
|
if len(days) >= 2:
|
|
strategy = AnalystRevisionStrategy(snapshots)
|
|
args = dict(member=stock["member"], start=str(days[0].date()), end=str(days[-1].date()))
|
|
plan = prepare_strategy(protocol["baseline"], "us", stock["panels"], stock["stocks"],
|
|
strategy=strategy, **args)
|
|
active = strategy.active.loc[days[0]:days[-1]]
|
|
result["active_signal_sessions"] = int(active.sum())
|
|
result["snapshot_collection_dates"] = len({_utc(d["created_at"]).date() for d in snapshots
|
|
if _utc(d["created_at"]) <= session_deadline(days[-1])})
|
|
if active.any():
|
|
start = active.index[active][0] + pd.offsets.BDay(1)
|
|
# Shifted signals trade one session after the collection cutoff.
|
|
live_days = plan["prices"].index[plan["prices"].index >= start]
|
|
result["evaluated_sessions"] = len(live_days)
|
|
if len(live_days) >= 2:
|
|
args["start"] = str(live_days[0].date())
|
|
plan = prepare_strategy(protocol["baseline"], "us", stock["panels"], stock["stocks"],
|
|
strategy=strategy, **args)
|
|
base = prepare_strategy(protocol["baseline"], "us", stock["panels"], stock["stocks"], **args)
|
|
enough = (int(active.sum()) >= 252 and result["snapshot_collection_dates"] >= 20
|
|
and float(active.mean()) >= MIN_COVERAGE)
|
|
returns = {}
|
|
for cost in protocol["costs_per_side"]:
|
|
pair = {}
|
|
for name, p in (("base", base), ("eps_revisions", plan)):
|
|
run = replay_prepared(p, 10000, cost, 2.0, fee_base=1.88, fee_per_share=0.009)
|
|
r = run["equity"].pct_change().dropna()
|
|
returns[f"{name}_{cost}"] = r
|
|
pair[name] = {"orders": len(run["trade_log"]),
|
|
"fees": sum(t["commission"] for t in run["trade_log"]),
|
|
"total_return": float(run["equity"].iloc[-1] / run["equity"].iloc[0] - 1)}
|
|
if enough:
|
|
pair[name].update(metrics.raw_summary(run["equity"]))
|
|
result["comparisons"].append({"cost": cost, **pair})
|
|
pd.DataFrame(returns).to_csv(destination / "returns.csv")
|
|
if enough:
|
|
passed = all(c["eps_revisions"]["annualizedReturn"] > c["base"]["annualizedReturn"]
|
|
and c["eps_revisions"]["sharpeRatio"] >= c["base"]["sharpeRatio"] + 0.10
|
|
and c["eps_revisions"]["maxDrawdown"] >= c["base"]["maxDrawdown"] - 0.02
|
|
for c in result["comparisons"])
|
|
result["verdict"] = "FORWARD CANDIDATE" if passed else "RETAIN BASELINE"
|
|
append_trials(destination / "trials.jsonl", result["comparisons"], protocol)
|
|
result["boundary"] = "Insufficient forward evidence never receives annualized performance or a promotion verdict."
|
|
(destination / "results.json").write_text(json.dumps(result, indent=2, allow_nan=False))
|
|
print(json.dumps(result, indent=2), flush=True)
|
|
print(f"Saved -> {destination}", flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = ap.add_subparsers(dest="action", required=True)
|
|
p = sub.add_parser("collect")
|
|
g = p.add_mutually_exclusive_group(required=True)
|
|
g.add_argument("--candidates", type=Path)
|
|
g.add_argument("--tickers", nargs="+")
|
|
p.add_argument("--directory", type=Path, default=SNAPSHOTS)
|
|
p = sub.add_parser("evaluate")
|
|
p.add_argument("--directory", type=Path, default=SNAPSHOTS)
|
|
p.add_argument("--output", type=Path)
|
|
args = ap.parse_args()
|
|
if args.action == "collect":
|
|
tickers = args.tickers if args.tickers else [r["ticker"] for r in json.loads(args.candidates.read_text())["candidates"]]
|
|
collect(tickers, args.directory)
|
|
else:
|
|
evaluate(args.directory, args.output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|