gahow/replaysim
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ReplayServe Frontier vLLM alignment report

Browse Source
This commit is contained in:
Gahow Wang
2026-06-25 17:10:30 +08:00
commit a99bd00782
63 changed files with 17033 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
docs/assets/frontier_vllm_alignment/completion_prefix.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 129 KiB

10
docs/assets/frontier_vllm_alignment/frontier_vllm_alignment.csv Normal file
View File

@@ -0,0 +1,10 @@
run_id,label,tp,request_count,scale_label,scale_value,fixture,kv_blocks,frontier_completed,frontier_total,frontier_complete,vllm_completed,vllm_total,frontier_preemptions,vllm_preemptions,frontier_prefix_hit,vllm_prefix_hit,prefix_hit_delta,frontier_rps,vllm_rps,rps_ratio,frontier_total_tps,vllm_total_tps,total_tps_ratio,frontier_decode_tps,vllm_decode_tps,decode_tps_ratio,frontier_ttft_p50_s,vllm_ttft_p50_s,ttft_p50_ratio,frontier_ttft_p95_s,vllm_ttft_p95_s,ttft_p95_ratio,frontier_tpot_p50_s,vllm_tpot_p50_s,tpot_p50_ratio,frontier_tpot_p95_s,vllm_tpot_p95_s,tpot_p95_ratio,frontier_e2e_p50_s,vllm_e2e_p50_s,e2e_p50_ratio,frontier_e2e_p95_s,vllm_e2e_p95_s,e2e_p95_ratio,notes
tp1_n100_scale1,TP1 N100 raw,1,100,raw,1,coder_100,15281,96,100,false,100,100,0,8,0.2487845616,0.2510820686,-0.002297507075,0.4048148795,0.6879880691,0.588403924,2348.908821,3832.320581,0.6129207541,347.7992338,567.4456795,0.6129207541,0.9087481136,4.503025495,0.201808343,12.76295815,29.06046906,0.4391862402,0.05688966428,0.06608134396,0.8609035603,0.1456880793,0.6211491471,0.2345460505,30.93928316,41.84076733,0.7394530534,119.6361376,97.36622969,1.228723121,Frontier incomplete before lifecycle fix; included as TP1 100-request baseline.
tp1_n500_scale1,TP1 N500 raw,1,500,raw,1,coder_500,15281,439,500,false,500,500,0,63,0.1192374692,0.3868498695,-0.2676124002,0.660990472,0.8401719451,0.7867323776,4733.748762,5282.903731,0.896050544,656.2204998,732.3476384,0.896050544,136.7755789,185.6581683,0.7367064976,340.2371222,375.8950067,0.9051387119,0.05643274739,0.04975253624,1.134268756,0.08942839773,0.0918798539,0.9733188935,177.7998574,224.2697872,0.7927945162,397.29145,417.3562933,0.9519239469,Frontier incomplete; useful as high-pressure stress signal.
tp1_n200_scale0667,TP1 N200 scale 0.667,1,200,0.667,0.6666666667,coder_200_ts0667,15281,176,200,false,200,200,0,26,0.170276008,0.2697549478,-0.09947893984,0.5830903706,0.8236788215,0.7079098737,3913.437526,4864.778909,0.8044430383,593.287826,737.51378,0.8044430383,20.58014532,34.56323652,0.595434554,96.71793818,120.8039818,0.800618794,0.05837096651,0.05145431897,1.13442307,0.235894569,0.2534757496,0.9306395954,73.20731169,83.6219905,0.875455263,189.2402903,183.726977,1.030008186,Dense-arrival run; Frontier incomplete before lifecycle fix.
tp1_n200_scale2,TP1 N200 scale 2,1,200,2,2,coder_200_ts2,15281,200,200,true,200,200,33,43,0.23134169,0.2697549478,-0.03841325784,0.5936627655,0.8029813635,0.7393232178,3506.267279,4742.53641,0.7393232178,531.5597036,718.9814831,0.7393232178,9.595321274,9.216767096,1.041072338,77.50341053,69.21141595,1.119806747,0.05421362546,0.04970337519,1.09074334,0.06653162646,0.06863309532,0.9693811149,61.45769412,55.00248734,1.117362088,174.4840836,142.3375087,1.225847531,After Frontier decode-preemption lifecycle fix.
tp1_n200_scale3,TP1 N200 scale 3,1,200,3,3,coder_200_ts3,15281,200,200,true,200,200,20,16,0.2176751278,0.2697549478,-0.05207982007,0.5739781652,0.7802265504,0.735655772,3390.00688,4608.142843,0.735655772,513.9343094,698.607051,0.735655772,1.001474116,1.166151478,0.8587856162,45.9466567,32.25842447,1.424330464,0.05339333437,0.04616159714,1.156661331,0.06861254671,0.0713836296,0.9611804148,44.76058145,33.21267588,1.34769573,154.5483135,122.7887113,1.258652459,After Frontier decode-preemption lifecycle fix.
tp2_n200_scale2,TP2 N200 scale 2,2,200,2,2,coder_200_ts2,69055,200,200,true,200,200,0,0,0.2697549478,0.2697549478,0,0.7756823572,1.277818683,0.607036325,4581.304111,7547.001591,0.607036325,694.5382258,1144.14607,0.607036325,0.2690959621,0.225119116,1.195349231,6.744624223,0.715071776,9.432094022,0.04295527658,0.03004499679,1.429698158,0.05288764732,0.04340382318,1.218502046,26.05122482,16.44861007,1.583794905,106.7591651,72.5347179,1.471835394,Uses true-mixed TP2/TP4 attention profile.
tp2_n200_scale3,TP2 N200 scale 3,2,200,3,3,coder_200_ts3,69055,200,200,true,200,200,0,0,0.2697549478,0.2697549478,0,0.6877705321,1.088050278,0.6321128225,4062.082806,6426.199028,0.6321128225,615.8228567,974.2293382,0.6321128225,0.1341535495,0.153530943,0.8737883511,0.5741378218,0.6270455511,0.9156237864,0.03937896849,0.01905767256,2.06630523,0.04670767225,0.02799082097,1.668678182,21.78596494,9.956003374,2.188223941,101.5918393,53.98348621,1.881905864,Uses true-mixed TP2/TP4 attention profile.
tp4_n200_scale2,TP4 N200 scale 2,4,200,2,2,coder_200_ts2,177077,200,200,true,200,200,0,0,0.2697549478,0.2697549478,0,0.8525337931,1.536203537,0.5549614829,5035.200987,9073.063884,0.5549614829,763.350233,1375.501285,0.5549614829,0.09755515041,0.1704972619,0.5721801589,0.3856872342,1.419861408,0.2716372401,0.03366585047,0.01634437735,2.059781767,0.03838265621,0.02831690026,1.355468143,18.65216282,9.260885488,2.014079846,84.93775414,43.62188903,1.947136083,Uses true-mixed TP2/TP4 attention profile.
tp4_n200_scale3,TP4 N200 scale 3,4,200,3,3,coder_200_ts3,177077,200,200,true,200,200,0,0,0.2697549478,0.2697549478,0,0.7373665172,1.253504493,0.5882440162,4355.004629,7403.398096,0.5882440162,660.2306059,1122.375388,0.5882440162,0.08859749135,0.100106278,0.885034317,0.3458954617,0.3184188101,1.086290919,0.03106778109,0.009410284212,3.301471071,0.03578285082,0.01279276668,2.79711588,16.90291941,5.54948732,3.045852424,83.00995365,27.86907583,2.978568581,Uses true-mixed TP2/TP4 attention profile.
1 run_id label tp request_count scale_label scale_value fixture kv_blocks frontier_completed frontier_total frontier_complete vllm_completed vllm_total frontier_preemptions vllm_preemptions frontier_prefix_hit vllm_prefix_hit prefix_hit_delta frontier_rps vllm_rps rps_ratio frontier_total_tps vllm_total_tps total_tps_ratio frontier_decode_tps vllm_decode_tps decode_tps_ratio frontier_ttft_p50_s vllm_ttft_p50_s ttft_p50_ratio frontier_ttft_p95_s vllm_ttft_p95_s ttft_p95_ratio frontier_tpot_p50_s vllm_tpot_p50_s tpot_p50_ratio frontier_tpot_p95_s vllm_tpot_p95_s tpot_p95_ratio frontier_e2e_p50_s vllm_e2e_p50_s e2e_p50_ratio frontier_e2e_p95_s vllm_e2e_p95_s e2e_p95_ratio notes
2 tp1_n100_scale1 TP1 N100 raw 1 100 raw 1 coder_100 15281 96 100 false 100 100 0 8 0.2487845616 0.2510820686 -0.002297507075 0.4048148795 0.6879880691 0.588403924 2348.908821 3832.320581 0.6129207541 347.7992338 567.4456795 0.6129207541 0.9087481136 4.503025495 0.201808343 12.76295815 29.06046906 0.4391862402 0.05688966428 0.06608134396 0.8609035603 0.1456880793 0.6211491471 0.2345460505 30.93928316 41.84076733 0.7394530534 119.6361376 97.36622969 1.228723121 Frontier incomplete before lifecycle fix; included as TP1 100-request baseline.
3 tp1_n500_scale1 TP1 N500 raw 1 500 raw 1 coder_500 15281 439 500 false 500 500 0 63 0.1192374692 0.3868498695 -0.2676124002 0.660990472 0.8401719451 0.7867323776 4733.748762 5282.903731 0.896050544 656.2204998 732.3476384 0.896050544 136.7755789 185.6581683 0.7367064976 340.2371222 375.8950067 0.9051387119 0.05643274739 0.04975253624 1.134268756 0.08942839773 0.0918798539 0.9733188935 177.7998574 224.2697872 0.7927945162 397.29145 417.3562933 0.9519239469 Frontier incomplete; useful as high-pressure stress signal.
4 tp1_n200_scale0667 TP1 N200 scale 0.667 1 200 0.667 0.6666666667 coder_200_ts0667 15281 176 200 false 200 200 0 26 0.170276008 0.2697549478 -0.09947893984 0.5830903706 0.8236788215 0.7079098737 3913.437526 4864.778909 0.8044430383 593.287826 737.51378 0.8044430383 20.58014532 34.56323652 0.595434554 96.71793818 120.8039818 0.800618794 0.05837096651 0.05145431897 1.13442307 0.235894569 0.2534757496 0.9306395954 73.20731169 83.6219905 0.875455263 189.2402903 183.726977 1.030008186 Dense-arrival run; Frontier incomplete before lifecycle fix.
5 tp1_n200_scale2 TP1 N200 scale 2 1 200 2 2 coder_200_ts2 15281 200 200 true 200 200 33 43 0.23134169 0.2697549478 -0.03841325784 0.5936627655 0.8029813635 0.7393232178 3506.267279 4742.53641 0.7393232178 531.5597036 718.9814831 0.7393232178 9.595321274 9.216767096 1.041072338 77.50341053 69.21141595 1.119806747 0.05421362546 0.04970337519 1.09074334 0.06653162646 0.06863309532 0.9693811149 61.45769412 55.00248734 1.117362088 174.4840836 142.3375087 1.225847531 After Frontier decode-preemption lifecycle fix.
6 tp1_n200_scale3 TP1 N200 scale 3 1 200 3 3 coder_200_ts3 15281 200 200 true 200 200 20 16 0.2176751278 0.2697549478 -0.05207982007 0.5739781652 0.7802265504 0.735655772 3390.00688 4608.142843 0.735655772 513.9343094 698.607051 0.735655772 1.001474116 1.166151478 0.8587856162 45.9466567 32.25842447 1.424330464 0.05339333437 0.04616159714 1.156661331 0.06861254671 0.0713836296 0.9611804148 44.76058145 33.21267588 1.34769573 154.5483135 122.7887113 1.258652459 After Frontier decode-preemption lifecycle fix.
7 tp2_n200_scale2 TP2 N200 scale 2 2 200 2 2 coder_200_ts2 69055 200 200 true 200 200 0 0 0.2697549478 0.2697549478 0 0.7756823572 1.277818683 0.607036325 4581.304111 7547.001591 0.607036325 694.5382258 1144.14607 0.607036325 0.2690959621 0.225119116 1.195349231 6.744624223 0.715071776 9.432094022 0.04295527658 0.03004499679 1.429698158 0.05288764732 0.04340382318 1.218502046 26.05122482 16.44861007 1.583794905 106.7591651 72.5347179 1.471835394 Uses true-mixed TP2/TP4 attention profile.
8 tp2_n200_scale3 TP2 N200 scale 3 2 200 3 3 coder_200_ts3 69055 200 200 true 200 200 0 0 0.2697549478 0.2697549478 0 0.6877705321 1.088050278 0.6321128225 4062.082806 6426.199028 0.6321128225 615.8228567 974.2293382 0.6321128225 0.1341535495 0.153530943 0.8737883511 0.5741378218 0.6270455511 0.9156237864 0.03937896849 0.01905767256 2.06630523 0.04670767225 0.02799082097 1.668678182 21.78596494 9.956003374 2.188223941 101.5918393 53.98348621 1.881905864 Uses true-mixed TP2/TP4 attention profile.
9 tp4_n200_scale2 TP4 N200 scale 2 4 200 2 2 coder_200_ts2 177077 200 200 true 200 200 0 0 0.2697549478 0.2697549478 0 0.8525337931 1.536203537 0.5549614829 5035.200987 9073.063884 0.5549614829 763.350233 1375.501285 0.5549614829 0.09755515041 0.1704972619 0.5721801589 0.3856872342 1.419861408 0.2716372401 0.03366585047 0.01634437735 2.059781767 0.03838265621 0.02831690026 1.355468143 18.65216282 9.260885488 2.014079846 84.93775414 43.62188903 1.947136083 Uses true-mixed TP2/TP4 attention profile.
10 tp4_n200_scale3 TP4 N200 scale 3 4 200 3 3 coder_200_ts3 177077 200 200 true 200 200 0 0 0.2697549478 0.2697549478 0 0.7373665172 1.253504493 0.5882440162 4355.004629 7403.398096 0.5882440162 660.2306059 1122.375388 0.5882440162 0.08859749135 0.100106278 0.885034317 0.3458954617 0.3184188101 1.086290919 0.03106778109 0.009410284212 3.301471071 0.03578285082 0.01279276668 2.79711588 16.90291941 5.54948732 3.045852424 83.00995365 27.86907583 2.978568581 Uses true-mixed TP2/TP4 attention profile.

434
docs/assets/frontier_vllm_alignment/frontier_vllm_alignment.json Normal file
View File

@@ -0,0 +1,434 @@
[
{
"decode_tps_ratio": 0.6129207541251469,
"e2e_p50_ratio": 0.7394530533987747,
"e2e_p95_ratio": 1.2287231205931113,
"fixture": "coder_100",
"frontier_complete": false,
"frontier_completed": 96,
"frontier_decode_tps": 347.79923381681954,
"frontier_e2e_p50_s": 30.939283157873398,
"frontier_e2e_p95_s": 119.6361375789676,
"frontier_preemptions": 0,
"frontier_prefix_hit": 0.24878456156190046,
"frontier_rps": 0.4048148795016268,
"frontier_total": 100,
"frontier_total_tps": 2348.908820556559,
"frontier_tpot_p50_s": 0.056889664283438265,
"frontier_tpot_p95_s": 0.14568807925543142,
"frontier_ttft_p50_s": 0.9087481136376141,
"frontier_ttft_p95_s": 12.762958146117297,
"kv_blocks": 15281,
"label": "TP1 N100 raw",
"notes": "Frontier incomplete before lifecycle fix; included as TP1 100-request baseline.",
"prefix_hit_delta": -0.0022975070751777016,
"request_count": 100,
"rps_ratio": 0.5884039239601411,
"run_id": "tp1_n100_scale1",
"scale_label": "raw",
"scale_value": 1.0,
"total_tps_ratio": 0.6129207541251469,
"tp": 1,
"tpot_p50_ratio": 0.8609035602986401,
"tpot_p95_ratio": 0.23454605053898236,
"ttft_p50_ratio": 0.20180834300191677,
"ttft_p95_ratio": 0.439186240241972,
"vllm_completed": 100,
"vllm_decode_tps": 567.445679520595,
"vllm_e2e_p50_s": 41.84076732886024,
"vllm_e2e_p95_s": 97.36622968502343,
"vllm_preemptions": 8,
"vllm_prefix_hit": 0.25108206863707816,
"vllm_rps": 0.6879880691092217,
"vllm_total": 100,
"vllm_total_tps": 3832.3205810011714,
"vllm_tpot_p50_s": 0.06608134395878643,
"vllm_tpot_p95_s": 0.6211491471318447,
"vllm_ttft_p50_s": 4.503025494981557,
"vllm_ttft_p95_s": 29.060469059972093
},
{
"decode_tps_ratio": 0.8960505440100501,
"e2e_p50_ratio": 0.7927945162118318,
"e2e_p95_ratio": 0.951923946910999,
"fixture": "coder_500",
"frontier_complete": false,
"frontier_completed": 439,
"frontier_decode_tps": 656.2204997652797,
"frontier_e2e_p50_s": 177.7998574092898,
"frontier_e2e_p95_s": 397.29145000151055,
"frontier_preemptions": 0,
"frontier_prefix_hit": 0.11923746923408568,
"frontier_rps": 0.6609904720097601,
"frontier_total": 500,
"frontier_total_tps": 4733.748762075876,
"frontier_tpot_p50_s": 0.05643274739314083,
"frontier_tpot_p95_s": 0.08942839772817235,
"frontier_ttft_p50_s": 136.77557892500107,
"frontier_ttft_p95_s": 340.237122196321,
"kv_blocks": 15281,
"label": "TP1 N500 raw",
"notes": "Frontier incomplete; useful as high-pressure stress signal.",
"prefix_hit_delta": -0.2676124002320734,
"request_count": 500,
"rps_ratio": 0.786732377640824,
"run_id": "tp1_n500_scale1",
"scale_label": "raw",
"scale_value": 1.0,
"total_tps_ratio": 0.8960505440100502,
"tp": 1,
"tpot_p50_ratio": 1.134268756171416,
"tpot_p95_ratio": 0.9733188934802052,
"ttft_p50_ratio": 0.736706497600023,
"ttft_p95_ratio": 0.9051387119015346,
"vllm_completed": 500,
"vllm_decode_tps": 732.3476383692921,
"vllm_e2e_p50_s": 224.26978715602309,
"vllm_e2e_p95_s": 417.3562933159992,
"vllm_preemptions": 63,
"vllm_prefix_hit": 0.38684986946615907,
"vllm_rps": 0.8401719451179492,
"vllm_total": 500,
"vllm_total_tps": 5282.903730956031,
"vllm_tpot_p50_s": 0.049752536236317216,
"vllm_tpot_p95_s": 0.09187985389702198,
"vllm_ttft_p50_s": 185.6581683079712,
"vllm_ttft_p95_s": 375.8950067239348
},
{
"decode_tps_ratio": 0.8044430383408974,
"e2e_p50_ratio": 0.8754552629577944,
"e2e_p95_ratio": 1.030008185534932,
"fixture": "coder_200_ts0667",
"frontier_complete": false,
"frontier_completed": 176,
"frontier_decode_tps": 593.287826008356,
"frontier_e2e_p50_s": 73.20731168652793,
"frontier_e2e_p95_s": 189.24029025053343,
"frontier_preemptions": 0,
"frontier_prefix_hit": 0.17027600800712456,
"frontier_rps": 0.5830903705506575,
"frontier_total": 200,
"frontier_total_tps": 3913.43752605849,
"frontier_tpot_p50_s": 0.05837096651496554,
"frontier_tpot_p95_s": 0.23589456903741046,
"frontier_ttft_p50_s": 20.58014532403832,
"frontier_ttft_p95_s": 96.7179381828816,
"kv_blocks": 15281,
"label": "TP1 N200 scale 0.667",
"notes": "Dense-arrival run; Frontier incomplete before lifecycle fix.",
"prefix_hit_delta": -0.09947893983522305,
"request_count": 200,
"rps_ratio": 0.7079098737399896,
"run_id": "tp1_n200_scale0667",
"scale_label": "0.667",
"scale_value": 0.6666666666666666,
"total_tps_ratio": 0.8044430383408974,
"tp": 1,
"tpot_p50_ratio": 1.1344230703885074,
"tpot_p95_ratio": 0.930639595403931,
"ttft_p50_ratio": 0.5954345540217358,
"ttft_p95_ratio": 0.800618794003408,
"vllm_completed": 200,
"vllm_decode_tps": 737.5137800085473,
"vllm_e2e_p50_s": 83.62199050490744,
"vllm_e2e_p95_s": 183.7269770358689,
"vllm_preemptions": 26,
"vllm_prefix_hit": 0.2697549478423476,
"vllm_rps": 0.8236788215286605,
"vllm_total": 200,
"vllm_total_tps": 4864.778908559713,
"vllm_tpot_p50_s": 0.051454318973762715,
"vllm_tpot_p95_s": 0.2534757495838373,
"vllm_ttft_p50_s": 34.563236522022635,
"vllm_ttft_p95_s": 120.80398175423034
},
{
"decode_tps_ratio": 0.7393232177681209,
"e2e_p50_ratio": 1.1173620884379967,
"e2e_p95_ratio": 1.2258475306262637,
"fixture": "coder_200_ts2",
"frontier_complete": true,
"frontier_completed": 200,
"frontier_decode_tps": 531.5597035900641,
"frontier_e2e_p50_s": 61.45769412455945,
"frontier_e2e_p95_s": 174.48408358603848,
"frontier_preemptions": 33,
"frontier_prefix_hit": 0.23134168999974056,
"frontier_rps": 0.5936627654877362,
"frontier_total": 200,
"frontier_total_tps": 3506.267279013048,
"frontier_tpot_p50_s": 0.054213625462090735,
"frontier_tpot_p95_s": 0.06653162646338621,
"frontier_ttft_p50_s": 9.595321273711544,
"frontier_ttft_p95_s": 77.50341053197451,
"kv_blocks": 15281,
"label": "TP1 N200 scale 2",
"notes": "After Frontier decode-preemption lifecycle fix.",
"prefix_hit_delta": -0.038413257842607046,
"request_count": 200,
"rps_ratio": 0.7393232177681209,
"run_id": "tp1_n200_scale2",
"scale_label": "2",
"scale_value": 2.0,
"total_tps_ratio": 0.7393232177681209,
"tp": 1,
"tpot_p50_ratio": 1.0907433399899442,
"tpot_p95_ratio": 0.9693811149298648,
"ttft_p50_ratio": 1.0410723384685256,
"ttft_p95_ratio": 1.1198067467817787,
"vllm_completed": 200,
"vllm_decode_tps": 718.9814830849542,
"vllm_e2e_p50_s": 55.002487340942025,
"vllm_e2e_p95_s": 142.3375087250024,
"vllm_preemptions": 43,
"vllm_prefix_hit": 0.2697549478423476,
"vllm_rps": 0.8029813635231063,
"vllm_total": 200,
"vllm_total_tps": 4742.53640998563,
"vllm_tpot_p50_s": 0.049703375188695206,
"vllm_tpot_p95_s": 0.06863309532102842,
"vllm_ttft_p50_s": 9.216767095960677,
"vllm_ttft_p95_s": 69.2114159471821
},
{
"decode_tps_ratio": 0.7356557719569122,
"e2e_p50_ratio": 1.3476957295017153,
"e2e_p95_ratio": 1.258652459348984,
"fixture": "coder_200_ts3",
"frontier_complete": true,
"frontier_completed": 200,
"frontier_decode_tps": 513.9343093668691,
"frontier_e2e_p50_s": 44.76058145123308,
"frontier_e2e_p95_s": 154.54831351855702,
"frontier_preemptions": 20,
"frontier_prefix_hit": 0.21767512777477313,
"frontier_rps": 0.573978165231764,
"frontier_total": 200,
"frontier_total_tps": 3390.0068803652352,
"frontier_tpot_p50_s": 0.053393334371887605,
"frontier_tpot_p95_s": 0.06861254670772189,
"frontier_ttft_p50_s": 1.0014741156186515,
"frontier_ttft_p95_s": 45.94665669959886,
"kv_blocks": 15281,
"label": "TP1 N200 scale 3",
"notes": "After Frontier decode-preemption lifecycle fix.",
"prefix_hit_delta": -0.05207982006757447,
"request_count": 200,
"rps_ratio": 0.7356557719569123,
"run_id": "tp1_n200_scale3",
"scale_label": "3",
"scale_value": 3.0,
"total_tps_ratio": 0.7356557719569123,
"tp": 1,
"tpot_p50_ratio": 1.1566613307426805,
"tpot_p95_ratio": 0.9611804148017213,
"ttft_p50_ratio": 0.8587856162345445,
"ttft_p95_ratio": 1.4243304641052532,
"vllm_completed": 200,
"vllm_decode_tps": 698.607050957755,
"vllm_e2e_p50_s": 33.2126758818049,
"vllm_e2e_p95_s": 122.78871134808287,
"vllm_preemptions": 16,
"vllm_prefix_hit": 0.2697549478423476,
"vllm_rps": 0.7802265503945264,
"vllm_total": 200,
"vllm_total_tps": 4608.1428428781355,
"vllm_tpot_p50_s": 0.04616159713544178,
"vllm_tpot_p95_s": 0.07138362959869063,
"vllm_ttft_p50_s": 1.1661514779552817,
"vllm_ttft_p95_s": 32.25842447206378
},
{
"decode_tps_ratio": 0.6070363250137228,
"e2e_p50_ratio": 1.5837949050918096,
"e2e_p95_ratio": 1.4718353941122981,
"fixture": "coder_200_ts2",
"frontier_complete": true,
"frontier_completed": 200,
"frontier_decode_tps": 694.538225813865,
"frontier_e2e_p50_s": 26.05122481685102,
"frontier_e2e_p95_s": 106.75916510714146,
"frontier_preemptions": 0,
"frontier_prefix_hit": 0.2697549478423476,
"frontier_rps": 0.7756823572006221,
"frontier_total": 200,
"frontier_total_tps": 4581.304110804026,
"frontier_tpot_p50_s": 0.042955276577521156,
"frontier_tpot_p95_s": 0.05288764732371923,
"frontier_ttft_p50_s": 0.2690959621493789,
"frontier_ttft_p95_s": 6.744624223172184,
"kv_blocks": 69055,
"label": "TP2 N200 scale 2",
"notes": "Uses true-mixed TP2/TP4 attention profile.",
"prefix_hit_delta": 0.0,
"request_count": 200,
"rps_ratio": 0.6070363250137228,
"run_id": "tp2_n200_scale2",
"scale_label": "2",
"scale_value": 2.0,
"total_tps_ratio": 0.6070363250137228,
"tp": 2,
"tpot_p50_ratio": 1.4296981582601855,
"tpot_p95_ratio": 1.218502045500008,
"ttft_p50_ratio": 1.1953492307083635,
"ttft_p95_ratio": 9.432094021900193,
"vllm_completed": 200,
"vllm_decode_tps": 1144.1460703330465,
"vllm_e2e_p50_s": 16.448610065039247,
"vllm_e2e_p95_s": 72.53471789998002,
"vllm_preemptions": 0,
"vllm_prefix_hit": 0.2697549478423476,
"vllm_rps": 1.2778186827338327,
"vllm_total": 200,
"vllm_total_tps": 7547.001591215254,
"vllm_tpot_p50_s": 0.030044996791346416,
"vllm_tpot_p95_s": 0.043403823177019754,
"vllm_ttft_p50_s": 0.22511911601759493,
"vllm_ttft_p95_s": 0.7150717759504914
},
{
"decode_tps_ratio": 0.6321128225155744,
"e2e_p50_ratio": 2.1882239414176055,
"e2e_p95_ratio": 1.8819058641979227,
"fixture": "coder_200_ts3",
"frontier_complete": true,
"frontier_completed": 200,
"frontier_decode_tps": 615.822856748031,
"frontier_e2e_p50_s": 21.785964943721574,
"frontier_e2e_p95_s": 101.59183927019191,
"frontier_preemptions": 0,
"frontier_prefix_hit": 0.2697549478423476,
"frontier_rps": 0.6877705321122985,
"frontier_total": 200,
"frontier_total_tps": 4062.0828059403734,
"frontier_tpot_p50_s": 0.0393789684875167,
"frontier_tpot_p95_s": 0.04670767224504207,
"frontier_ttft_p50_s": 0.13415354950526392,
"frontier_ttft_p95_s": 0.574137821753455,
"kv_blocks": 69055,
"label": "TP2 N200 scale 3",
"notes": "Uses true-mixed TP2/TP4 attention profile.",
"prefix_hit_delta": 0.0,
"request_count": 200,
"rps_ratio": 0.6321128225155745,
"run_id": "tp2_n200_scale3",
"scale_label": "3",
"scale_value": 3.0,
"total_tps_ratio": 0.6321128225155745,
"tp": 2,
"tpot_p50_ratio": 2.066305230245682,
"tpot_p95_ratio": 1.668678182045304,
"ttft_p50_ratio": 0.8737883511042303,
"ttft_p95_ratio": 0.9156237864420547,
"vllm_completed": 200,
"vllm_decode_tps": 974.229338201501,
"vllm_e2e_p50_s": 9.956003373954445,
"vllm_e2e_p95_s": 53.98348621092737,
"vllm_preemptions": 0,
"vllm_prefix_hit": 0.2697549478423476,
"vllm_rps": 1.0880502777577379,
"vllm_total": 200,
"vllm_total_tps": 6426.199028481642,
"vllm_tpot_p50_s": 0.01905767256023186,
"vllm_tpot_p95_s": 0.02799082096692385,
"vllm_ttft_p50_s": 0.15353094297461212,
"vllm_ttft_p95_s": 0.6270455510821193
},
{
"decode_tps_ratio": 0.554961482872708,
"e2e_p50_ratio": 2.0140798462106178,
"e2e_p95_ratio": 1.9471360828275543,
"fixture": "coder_200_ts2",
"frontier_complete": true,
"frontier_completed": 200,
"frontier_decode_tps": 763.3502329676248,
"frontier_e2e_p50_s": 18.65216281946347,
"frontier_e2e_p95_s": 84.93775413567799,
"frontier_preemptions": 0,
"frontier_prefix_hit": 0.2697549478423476,
"frontier_rps": 0.8525337930595883,
"frontier_total": 200,
"frontier_total_tps": 5035.200987216818,
"frontier_tpot_p50_s": 0.03366585046876145,
"frontier_tpot_p95_s": 0.03838265621202119,
"frontier_ttft_p50_s": 0.09755515041058871,
"frontier_ttft_p95_s": 0.3856872342439675,
"kv_blocks": 177077,
"label": "TP4 N200 scale 2",
"notes": "Uses true-mixed TP2/TP4 attention profile.",
"prefix_hit_delta": 0.0,
"request_count": 200,
"rps_ratio": 0.5549614828727081,
"run_id": "tp4_n200_scale2",
"scale_label": "2",
"scale_value": 2.0,
"total_tps_ratio": 0.5549614828727081,
"tp": 4,
"tpot_p50_ratio": 2.0597817670263323,
"tpot_p95_ratio": 1.3554681431066735,
"ttft_p50_ratio": 0.5721801588631308,
"ttft_p95_ratio": 0.27163724014492546,
"vllm_completed": 200,
"vllm_decode_tps": 1375.5012852715674,
"vllm_e2e_p50_s": 9.26088548800908,
"vllm_e2e_p95_s": 43.621889032190666,
"vllm_preemptions": 0,
"vllm_prefix_hit": 0.2697549478423476,
"vllm_rps": 1.5362035373095158,
"vllm_total": 200,
"vllm_total_tps": 9073.06388391597,
"vllm_tpot_p50_s": 0.016344377354773947,
"vllm_tpot_p95_s": 0.02831690025857032,
"vllm_ttft_p50_s": 0.1704972619190812,
"vllm_ttft_p95_s": 1.4198614079505205
},
{
"decode_tps_ratio": 0.5882440161960838,
"e2e_p50_ratio": 3.045852424279607,
"e2e_p95_ratio": 2.9785685814353515,
"fixture": "coder_200_ts3",
"frontier_complete": true,
"frontier_completed": 200,
"frontier_decode_tps": 660.2306058712501,
"frontier_e2e_p50_s": 16.902919407154563,
"frontier_e2e_p95_s": 83.00995364867583,
"frontier_preemptions": 0,
"frontier_prefix_hit": 0.2697549478423476,
"frontier_rps": 0.7373665172396945,
"frontier_total": 200,
"frontier_total_tps": 4355.004629460394,
"frontier_tpot_p50_s": 0.031067781092248118,
"frontier_tpot_p95_s": 0.035782850818878296,
"frontier_ttft_p50_s": 0.08859749134958328,
"frontier_ttft_p95_s": 0.3458954617429286,
"kv_blocks": 177077,
"label": "TP4 N200 scale 3",
"notes": "Uses true-mixed TP2/TP4 attention profile.",
"prefix_hit_delta": 0.0,
"request_count": 200,
"rps_ratio": 0.5882440161960838,
"run_id": "tp4_n200_scale3",
"scale_label": "3",
"scale_value": 3.0,
"total_tps_ratio": 0.5882440161960839,
"tp": 4,
"tpot_p50_ratio": 3.301471070786272,
"tpot_p95_ratio": 2.7971158799197804,
"ttft_p50_ratio": 0.8850343170011207,
"ttft_p95_ratio": 1.086290918512101,
"vllm_completed": 200,
"vllm_decode_tps": 1122.3753879226379,
"vllm_e2e_p50_s": 5.549487320007756,
"vllm_e2e_p95_s": 27.869075825903565,
"vllm_preemptions": 0,
"vllm_prefix_hit": 0.2697549478423476,
"vllm_rps": 1.2535044929278167,
"vllm_total": 200,
"vllm_total_tps": 7403.398095950554,
"vllm_tpot_p50_s": 0.00941028421153152,
"vllm_tpot_p95_s": 0.01279276667647553,
"vllm_ttft_p50_s": 0.1001062779687345,
"vllm_ttft_p95_s": 0.3184188101440668
}
]

BIN
docs/assets/frontier_vllm_alignment/latency_ratios.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 90 KiB

BIN
docs/assets/frontier_vllm_alignment/throughput_ratio.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 91 KiB

BIN
docs/assets/frontier_vllm_alignment/tp_scaling_total_tps.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 91 KiB