gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

f2b: regenerate CDF from production trace (1.3M sessions on dash0)

Browse Source 
Pulls 456 (rank%, cum%) sample points from the raw production trace at
dash0:/home/admin/cpfs/wjh/ali-trace/trace-glm5.1-formatted/051315-051317.jsonl,
cached locally so the figure is reproducible without ssh access. Sampled
anchors match the precomputed summary exactly:
  top 1% = 46.5%, top 5% = 66.5%, top 10% = 74.6%
plus newly readable points:
  top 25% = 87.5%, top 50% = 96.0%

Workload characterization is now consistent with the production
distribution rather than the small replay subset. Replay window CDF kept
as an overlay to show the same hockey-stick shape on the data §5 actually
uses.

- analysis/characterization/data/production_session_skew_cdf.json: cached
  sample points (29 KB), so the figure rebuilds locally
- scripts/plot_session_skew_cdf.py: now plots from the cache + replay raw
- MEETING.md / PAPER_OUTLINE.md: revert numbers to production trace,
  add top-25%/50% data points

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-27 10:41:53 +08:00
parent 22c4aa58e4
commit 1220da249c
5 changed files with 67 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
scripts/plot_session_skew_cdf.py
View File

@@ -1,12 +1,17 @@
#!/usr/bin/env python3
"""Plot a CDF of cumulative input-token mass by session rank.
Reads a JSONL trace (chat_id, session_id, input_length, ...), aggregates
per-session input_length, sorts sessions descending by total, and plots
cumulative fraction of input-token mass vs session-rank percentile.
Primary curve is the *production* trace
(``/home/admin/cpfs/wjh/ali-trace/trace-glm5.1-formatted/051315-051317.jsonl``
on dash0), which has 1.3 M sessions across 2.1 M records over a 7200 s
window. Because the full raw trace is not co-located with this repo, we
sample 456 (rank_pct, cum_pct) points on dash0 and cache the result in
``analysis/characterization/data/production_session_skew_cdf.json``. Any
top-K%% mass figure can be read off the resulting curve.
The figure replaces the previous discrete top-1%/5%/10% bars with a
continuous curve so any percentile can be read off directly.
The replay-trace CDF (``traces/w600_r0.0015_st30.jsonl``, n=274) is
overlaid for sanity — the replay window samples a thin slice of the head
so its top-1%% is lower, but the shape is preserved.
"""
from __future__ import annotations
@@ -19,66 +24,85 @@ import matplotlib.pyplot as plt
import numpy as np
def load_session_input_tokens(trace_path: Path) -> dict[str, int]:
def load_replay_cdf(trace_path: Path) -> tuple[np.ndarray, np.ndarray, int]:
totals: dict[str, int] = defaultdict(int)
with trace_path.open() as f:
for line in f:
row = json.loads(line)
totals[row["session_id"]] += int(row["input_length"])
return dict(totals)
n = len(totals)
sorted_vals = np.sort(np.array(list(totals.values())))[::-1]
cum = np.cumsum(sorted_vals) / sorted_vals.sum()
rank_pct = np.arange(1, n + 1) / n * 100
return rank_pct, cum * 100, n
def load_production_cdf(
cache_path: Path,
) -> tuple[np.ndarray, np.ndarray, int, dict[str, float]]:
d = json.loads(cache_path.read_text())
samples = d["samples"]
xs = np.array([s["rank_pct"] for s in samples])
ys = np.array([s["cum_pct"] for s in samples])
return xs, ys, d["n_sessions"], d["anchors_check"]
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"--trace",
"--replay-trace",
default="traces/w600_r0.0015_st30.jsonl",
help="JSONL trace path",
)
parser.add_argument(
"--out",
default="figs/f2b_session_skew.png",
help="Output figure path",
"--prod-cache",
default="analysis/characterization/data/production_session_skew_cdf.json",
)
parser.add_argument("--out", default="figs/f2b_session_skew.png")
args = parser.parse_args()
session_totals = load_session_input_tokens(Path(args.trace))
n_sessions = len(session_totals)
sorted_vals = np.sort(np.array(list(session_totals.values())))[::-1]
cum = np.cumsum(sorted_vals) / sorted_vals.sum()
rank_pct = np.arange(1, n_sessions + 1) / n_sessions * 100
prod_x, prod_y, prod_n, prod_anchors = load_production_cdf(Path(args.prod_cache))
replay_rank_pct, replay_cum_pct, replay_n = load_replay_cdf(Path(args.replay_trace))
marks = [1, 5, 10, 25, 50]
mark_idx = [int(np.ceil(n_sessions * p / 100)) - 1 for p in marks]
fig, ax = plt.subplots(figsize=(9, 5.5))
fig, ax = plt.subplots(figsize=(8, 5))
ax.plot(rank_pct, cum * 100, color="#2f6fab", lw=2.2,
label="cumulative input-token mass")
ax.plot([0, 100], [0, 100], color="#999", ls="--", lw=1,
label="uniform reference (y = x)")
ax.plot(
prod_x, prod_y,
color="#c44e52", lw=2.4,
label=f"production trace (n={prod_n:,} sessions, 456-pt sampled)",
)
for p, i in zip(marks, mark_idx):
y = cum[i] * 100
ax.scatter([p], [y], color="#c44e52", zorder=5, s=40)
annotate_pts = [1.0, 5.0, 10.0, 25.0, 50.0]
for p in annotate_pts:
y = float(np.interp(p, prod_x, prod_y))
ax.scatter([p], [y], color="#c44e52", s=55, zorder=5)
ax.annotate(
f"top {p}% → {y:.1f}%",
f"top {p:g}% → {y:.1f}%",
xy=(p, y),
xytext=(p + 2, y - 5),
fontsize=9,
color="#333",
xytext=(p + 2.5, y - 6),
fontsize=10,
color="#7a1d1d",
)
ax.plot(
replay_rank_pct, replay_cum_pct,
color="#2f6fab", lw=1.6,
alpha=0.85,
label=f"replay window (n={replay_n} sessions, raw CDF)",
)
ax.plot(
[0, 100], [0, 100],
color="#888", ls="--", lw=1,
label="uniform reference (y = x)",
)
ax.set_xlim(0, 100)
ax.set_ylim(0, 102)
ax.set_xlabel("Session rank percentile (top → bottom by input-token mass)")
ax.set_ylabel("Cumulative % of input-token mass")
ax.set_title(
f"Session input-token mass CDF "
f"(n={n_sessions} sessions, "
f"total={sorted_vals.sum() / 1e6:.1f} M tokens)"
)
ax.set_title("Session input-token mass CDF — Qwen3 production trace")
ax.grid(True, alpha=0.3)
ax.legend(loc="lower right", framealpha=0.9)
ax.legend(loc="lower right", framealpha=0.92, fontsize=9)
out_path = Path(args.out)
out_path.parent.mkdir(parents=True, exist_ok=True)