feat(analyzer): surface TTFT/TPOT as features and CDF figures

The gateway records three epoch-ms stamps per request (admitted, forwarded to
backend, backend first token). They reached *-raw.jsonl but stopped there:
features.csv had only total latency, and none of the figures covered
time-to-first-token.

Derive ttft_ms, ttft_e2e_ms and tpot_ms in one helper shared by the streaming
path (preparation.py) and the in-memory path (features.py) so the two cannot
drift, and add figures 14 (TTFT, backend vs end-to-end) and 15 (TPOT). The two
TTFT series are plotted together because their offset is the gateway admission
delay, which is then readable straight off the chart.

Rows missing a backend stamp and single-token responses are recorded as 0 and
filtered at plot time; left in, they would stack at the origin and pull every
percentile down. Figure 15 is a single panel: TPOT is near-deterministic
(p50 21.9, p90 23.0), so a p90 zoom renders a flat line and a vertical edge.

Verified by recomputing ttft_ms straight from the original SLS rows for all
2190 requests of a sample: 0 mismatches.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-11 06:31:48 +00:00
parent 7cb4cac56d
commit 825062f554
8 changed files with 95 additions and 6 deletions

View File

@@ -44,6 +44,8 @@ FIGURE_STEMS = [
"11_alive_kvcache_blocks_timeline",
"12_bucket_kvcache_reuse_ratio",
"13_session_cross_bucket_kvcache_miss",
"14_ttft_cdf",
"15_tpot_cdf",
]
@@ -112,6 +114,9 @@ def _load_request_metrics(path: Path) -> list[dict]:
"output_tokens": safe_int(row.get("output_tokens")),
"request_ready_time_ms": safe_int(row.get("request_ready_time_ms")),
"request_end_time_ms": safe_int(row.get("request_end_time_ms")),
"ttft_ms": safe_int(row.get("ttft_ms")),
"ttft_e2e_ms": safe_int(row.get("ttft_e2e_ms")),
"tpot_ms": safe_float(row.get("tpot_ms")),
"input_length_bucket": row.get("input_length_bucket", ""),
"theoretical_prompt_unit_length": safe_int(row.get("theoretical_prompt_unit_length")),
"theoretical_prefix_hit_blocks": safe_int(row.get("theoretical_prefix_hit_blocks")),
@@ -623,6 +628,42 @@ def _plot_session_cross_bucket_miss(fig_dir: Path, rows: list[dict]) -> None:
_save(fig, fig_dir, "13_session_cross_bucket_kvcache_miss")
def _plot_ttft_cdf(fig_dir: Path, request_rows: list[dict]) -> None:
# Requests missing either backend timestamp carry 0. Plotting those would pile
# a spike at the origin and drag every percentile down, so drop them.
backend_values = [row["ttft_ms"] for row in request_rows if row["ttft_ms"] > 0]
e2e_values = [row["ttft_e2e_ms"] for row in request_rows if row["ttft_e2e_ms"] > 0]
_plot_two_series_cdf_with_zoom(
fig_dir,
stem="14_ttft_cdf",
title="Time To First Token CDF",
xlabel="Milliseconds",
first_label="TTFT backend",
first_values=backend_values,
first_color=PALETTE["teal"],
second_label="TTFT end-to-end",
second_values=e2e_values,
second_color=PALETTE["red"],
zoom_quantile=0.90,
stats_labels=("mean", "p50", "p90", "p95", "p99"),
)
def _plot_tpot_cdf(fig_dir: Path, request_rows: list[dict]) -> None:
# A single-token response has no decode interval to average over and is
# reported as 0; it is not a zero-cost token.
values = [row["tpot_ms"] for row in request_rows if row["tpot_ms"] > 0]
_plot_single_cdf(
fig_dir,
stem="15_tpot_cdf",
title="Time Per Output Token CDF",
xlabel="Milliseconds per output token",
label="TPOT",
values=values,
color=PALETTE["green"],
)
def _write_manifest(fig_dir: Path, manifest: dict) -> None:
(fig_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
@@ -781,6 +822,14 @@ def render_figures(
progress.update(1)
progress.set_postfix(current="13_session_cross_bucket_kvcache_miss")
_plot_session_cross_bucket_miss(fig_root, session_bucket_rows)
if show_progress:
progress.update(1)
progress.set_postfix(current="14_ttft_cdf")
_plot_ttft_cdf(fig_root, request_rows)
if show_progress:
progress.update(1)
progress.set_postfix(current="15_tpot_cdf")
_plot_tpot_cdf(fig_root, request_rows)
_plot_session_gap_cdf(fig_root, session_rows_by_id)
if show_progress: