feat(analyzer): surface TTFT/TPOT as features and CDF figures
The gateway records three epoch-ms stamps per request (admitted, forwarded to backend, backend first token). They reached *-raw.jsonl but stopped there: features.csv had only total latency, and none of the figures covered time-to-first-token. Derive ttft_ms, ttft_e2e_ms and tpot_ms in one helper shared by the streaming path (preparation.py) and the in-memory path (features.py) so the two cannot drift, and add figures 14 (TTFT, backend vs end-to-end) and 15 (TPOT). The two TTFT series are plotted together because their offset is the gateway admission delay, which is then readable straight off the chart. Rows missing a backend stamp and single-token responses are recorded as 0 and filtered at plot time; left in, they would stack at the origin and pull every percentile down. Figure 15 is a single panel: TPOT is near-deterministic (p50 21.9, p90 23.0), so a p90 zoom renders a flat line and a vertical edge. Verified by recomputing ttft_ms straight from the original SLS rows for all 2190 requests of a sample: 0 mismatches. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -44,6 +44,8 @@ FIGURE_STEMS = [
|
||||
"11_alive_kvcache_blocks_timeline",
|
||||
"12_bucket_kvcache_reuse_ratio",
|
||||
"13_session_cross_bucket_kvcache_miss",
|
||||
"14_ttft_cdf",
|
||||
"15_tpot_cdf",
|
||||
]
|
||||
|
||||
|
||||
@@ -112,6 +114,9 @@ def _load_request_metrics(path: Path) -> list[dict]:
|
||||
"output_tokens": safe_int(row.get("output_tokens")),
|
||||
"request_ready_time_ms": safe_int(row.get("request_ready_time_ms")),
|
||||
"request_end_time_ms": safe_int(row.get("request_end_time_ms")),
|
||||
"ttft_ms": safe_int(row.get("ttft_ms")),
|
||||
"ttft_e2e_ms": safe_int(row.get("ttft_e2e_ms")),
|
||||
"tpot_ms": safe_float(row.get("tpot_ms")),
|
||||
"input_length_bucket": row.get("input_length_bucket", ""),
|
||||
"theoretical_prompt_unit_length": safe_int(row.get("theoretical_prompt_unit_length")),
|
||||
"theoretical_prefix_hit_blocks": safe_int(row.get("theoretical_prefix_hit_blocks")),
|
||||
@@ -623,6 +628,42 @@ def _plot_session_cross_bucket_miss(fig_dir: Path, rows: list[dict]) -> None:
|
||||
_save(fig, fig_dir, "13_session_cross_bucket_kvcache_miss")
|
||||
|
||||
|
||||
def _plot_ttft_cdf(fig_dir: Path, request_rows: list[dict]) -> None:
|
||||
# Requests missing either backend timestamp carry 0. Plotting those would pile
|
||||
# a spike at the origin and drag every percentile down, so drop them.
|
||||
backend_values = [row["ttft_ms"] for row in request_rows if row["ttft_ms"] > 0]
|
||||
e2e_values = [row["ttft_e2e_ms"] for row in request_rows if row["ttft_e2e_ms"] > 0]
|
||||
_plot_two_series_cdf_with_zoom(
|
||||
fig_dir,
|
||||
stem="14_ttft_cdf",
|
||||
title="Time To First Token CDF",
|
||||
xlabel="Milliseconds",
|
||||
first_label="TTFT backend",
|
||||
first_values=backend_values,
|
||||
first_color=PALETTE["teal"],
|
||||
second_label="TTFT end-to-end",
|
||||
second_values=e2e_values,
|
||||
second_color=PALETTE["red"],
|
||||
zoom_quantile=0.90,
|
||||
stats_labels=("mean", "p50", "p90", "p95", "p99"),
|
||||
)
|
||||
|
||||
|
||||
def _plot_tpot_cdf(fig_dir: Path, request_rows: list[dict]) -> None:
|
||||
# A single-token response has no decode interval to average over and is
|
||||
# reported as 0; it is not a zero-cost token.
|
||||
values = [row["tpot_ms"] for row in request_rows if row["tpot_ms"] > 0]
|
||||
_plot_single_cdf(
|
||||
fig_dir,
|
||||
stem="15_tpot_cdf",
|
||||
title="Time Per Output Token CDF",
|
||||
xlabel="Milliseconds per output token",
|
||||
label="TPOT",
|
||||
values=values,
|
||||
color=PALETTE["green"],
|
||||
)
|
||||
|
||||
|
||||
def _write_manifest(fig_dir: Path, manifest: dict) -> None:
|
||||
(fig_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
@@ -781,6 +822,14 @@ def render_figures(
|
||||
progress.update(1)
|
||||
progress.set_postfix(current="13_session_cross_bucket_kvcache_miss")
|
||||
_plot_session_cross_bucket_miss(fig_root, session_bucket_rows)
|
||||
if show_progress:
|
||||
progress.update(1)
|
||||
progress.set_postfix(current="14_ttft_cdf")
|
||||
_plot_ttft_cdf(fig_root, request_rows)
|
||||
if show_progress:
|
||||
progress.update(1)
|
||||
progress.set_postfix(current="15_tpot_cdf")
|
||||
_plot_tpot_cdf(fig_root, request_rows)
|
||||
_plot_session_gap_cdf(fig_root, session_rows_by_id)
|
||||
|
||||
if show_progress:
|
||||
|
||||
Reference in New Issue
Block a user