Gahow Wang
9b6091fe6e
Extends exp(c) (dispatch ablation, 1 round-robin policy) to the full 5-policy routing comparison, both modes on the SAME ttp trace (807 reqs, fresh vLLM/arm, dash0 8xH20). Confirms exp(c)'s prediction and finds something stronger: the dispatch mode FLIPS which policy wins. - thinktime helps every policy but helps LPWL most (TTFT p90 -40%, E2E mean -31% vs -3..-16% for the rest): tracets bursts punish prefill-spreading. - Ranking flip: tracets -> LPWL only ties unified_ab on TTFT p90 and is 3rd on E2E mean; thinktime -> LPWL is 1st on both (TTFT p90 -31%, best TPOT/balance, zero knobs) vs the tuned unified+A+B. - => benchmark agentic routing with thinktime; tracets' burst artifact erases LPWL's advantage. Caveat n=1: tracets ranking is run-sensitive (does not reproduce dash1 lpwl_5policy_600s.md), the thinktime advantage is the robust signal (appears in both environments). README + grouped-bar fig (figs/exp_d_policy_dispatch.png) + bench_report summaries in results/. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
69 lines
3.1 KiB
Python
69 lines
3.1 KiB
Python
"""exp (d): 5-policy routing under tracets vs thinktime dispatch.
|
|
|
|
Shows the ranking FLIP: under the faithful `thinktime` load the parameter-free
|
|
LPWL (leastwork) is the clear winner, but under `tracets` (think-collapse bursts)
|
|
its advantage disappears (it ties unified_ab on TTFT p90 and *loses* on E2E mean).
|
|
|
|
Reads the two bench_report summaries; writes v2/figs/exp_d_policy_dispatch.png.
|
|
Usage: python v2/exp_d_policy_dispatch/plot.py
|
|
"""
|
|
import json
|
|
import os
|
|
|
|
import matplotlib
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
|
|
HERE = os.path.dirname(__file__)
|
|
TC = json.load(open(os.path.join(HERE, "results/tracets.json")))
|
|
TT = json.load(open(os.path.join(HERE, "results/thinktime.json")))
|
|
|
|
# canonical order: LPWL first; pretty labels
|
|
ARMS = ["leastwork", "unified_ab", "unified_def", "lmetric", "sticky"]
|
|
LABEL = {"leastwork": "LPWL\n(leastwork)", "unified_ab": "unified\n+A+B",
|
|
"unified_def": "unified\ndefault", "lmetric": "LMetric", "sticky": "sticky"}
|
|
C_TC, C_TT = "#d62728", "#2ca02c" # tracets red / thinktime green (match exp_c)
|
|
|
|
|
|
def panel(ax, key, sub, title, ylab):
|
|
tc = [TC[a][key][sub] / 1000.0 for a in ARMS] # ms -> s
|
|
tt = [TT[a][key][sub] / 1000.0 for a in ARMS]
|
|
x = range(len(ARMS))
|
|
w = 0.38
|
|
b1 = ax.bar([i - w / 2 for i in x], tc, w, label="tracets (burst)", color=C_TC)
|
|
b2 = ax.bar([i + w / 2 for i in x], tt, w, label="thinktime (faithful)", color=C_TT)
|
|
for bars in (b1, b2):
|
|
for r in bars:
|
|
ax.text(r.get_x() + r.get_width() / 2, r.get_height(),
|
|
f"{r.get_height():.1f}", ha="center", va="bottom", fontsize=8)
|
|
ax.set_xticks(list(x)); ax.set_xticklabels([LABEL[a] for a in ARMS], fontsize=9)
|
|
ax.set_ylabel(ylab); ax.set_title(title, fontsize=11)
|
|
ax.grid(axis="y", alpha=.3)
|
|
ax.set_ylim(0, max(tc + tt) * 1.18)
|
|
# mark LPWL-thinktime as the winner (lowest green) in each panel
|
|
ax.annotate("LPWL wins\nunder thinktime", xy=(0 + w / 2, tt[0]),
|
|
xytext=(0.9, max(tc + tt) * 0.86), fontsize=8.5, color=C_TT,
|
|
ha="left", arrowprops=dict(arrowstyle="->", color=C_TT, lw=1.3))
|
|
return b1, b2
|
|
|
|
|
|
fig, (axL, axR) = plt.subplots(1, 2, figsize=(11.2, 4.6))
|
|
panel(axL, "ttft_ms", "p90", "TTFT p90 (lower = better)", "TTFT p90 (s)")
|
|
panel(axR, "e2e_ms", "mean", "E2E mean (lower = better)", "E2E mean (s)")
|
|
axL.legend(loc="upper left", fontsize=9)
|
|
fig.suptitle("5-policy routing: dispatch mode flips the ranking — "
|
|
"LPWL is best under faithful thinktime, only ties/loses under tracets bursts",
|
|
fontsize=11.5)
|
|
fig.tight_layout(rect=(0, 0, 1, 0.95))
|
|
out = os.path.join(HERE, "..", "figs", "exp_d_policy_dispatch.png")
|
|
fig.savefig(out, dpi=140)
|
|
print("wrote", os.path.normpath(out))
|
|
|
|
# also print the deltas the README cites
|
|
print("\npolicy TTFTp90 tc->tt E2Emean tc->tt")
|
|
for a in ARMS:
|
|
t1, t2 = TC[a]["ttft_ms"]["p90"], TT[a]["ttft_ms"]["p90"]
|
|
e1, e2 = TC[a]["e2e_ms"]["mean"], TT[a]["e2e_ms"]["mean"]
|
|
print(f"{a:<13} {t1/1000:5.1f}->{t2/1000:4.1f}s ({(t2-t1)/t1:+.0%}) "
|
|
f"{e1/1000:5.1f}->{e2/1000:4.1f}s ({(e2-e1)/e1:+.0%})")
|