plain [ { "rate_target": 0.5, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 424.285424329, "n_completed_total": 201, "n_after_warmup": 190, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 253.27869, "ttft_ms_p90": 290.162753, "ttft_ms_p99": 567.620172, "tpot_ms_p50": 6.3617311960784315, "tpot_ms_p90": 9.665774274509804, "tpot_ms_p99": 13.281522015686274, "e2e_ms_p50": 1880.48985, "e2e_ms_p90": 2745.165083, "e2e_ms_p99": 3789.202891, "throughput_effective_rps": 0.45862101064196187, "throughput_ratio": 0.9172420212839237, "inflight_p50": 2, "inflight_p90": 4, "phase": "A", "cell": "A_r0.5_4096x256" }, { "rate_target": 1.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 207.950502616, "n_completed_total": 201, "n_after_warmup": 183, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 259.603361, "ttft_ms_p90": 378.280699, "ttft_ms_p99": 524.687149, "tpot_ms_p50": 8.502388435294117, "tpot_ms_p90": 15.440471870588235, "tpot_ms_p99": 17.428640921568626, "e2e_ms_p50": 2493.103336, "e2e_ms_p90": 4232.406556, "e2e_ms_p99": 4871.38149, "throughput_effective_rps": 0.9244735304107706, "throughput_ratio": 0.9244735304107706, "inflight_p50": 3, "inflight_p90": 7, "phase": "A", "cell": "A_r1.0_4096x256" }, { "rate_target": 2.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 120.789826906, "n_completed_total": 223, "n_after_warmup": 205, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 293.365502, "ttft_ms_p90": 560.857942, "ttft_ms_p99": 840.781127, "tpot_ms_p50": 25.272439737254903, "tpot_ms_p90": 35.26223112156863, "tpot_ms_p99": 42.42426647843137, "e2e_ms_p50": 6825.599527, "e2e_ms_p90": 9263.482823, "e2e_ms_p99": 11140.719046, "throughput_effective_rps": 1.8503503952030986, "throughput_ratio": 0.9251751976015493, "inflight_p50": 13, "inflight_p90": 23, "phase": "A", "cell": "A_r2.0_4096x256" } ]noop_connector [ { "rate_target": 0.5, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 401.135882038, "n_completed_total": 219, "n_after_warmup": 211, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 264.899183, "ttft_ms_p90": 465.774583, "ttft_ms_p99": 683.105893, "tpot_ms_p50": 11.97862425490196, "tpot_ms_p90": 21.070080823529413, "tpot_ms_p99": 25.905328713725492, "e2e_ms_p50": 3317.307319, "e2e_ms_p90": 5651.525028, "e2e_ms_p99": 6872.18281, "throughput_effective_rps": 0.5394544701462617, "throughput_ratio": 1.0789089402925234, "inflight_p50": 3, "inflight_p90": 6, "phase": "A", "cell": "A_r0.5_4096x256" }, { "rate_target": 1.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 217.019525223, "n_completed_total": 205, "n_after_warmup": 195, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 317.569147, "ttft_ms_p90": 616.471598, "ttft_ms_p99": 871.3817, "tpot_ms_p50": 25.686870282352942, "tpot_ms_p90": 37.06463774901961, "tpot_ms_p99": 42.32126850588235, "e2e_ms_p50": 6867.08031, "e2e_ms_p90": 9953.52699, "e2e_ms_p99": 11064.851185, "throughput_effective_rps": 0.9419401372404239, "throughput_ratio": 0.9419401372404239, "inflight_p50": 7, "inflight_p90": 12, "phase": "A", "cell": "A_r1.0_4096x256" }, { "rate_target": 2.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 238.461594285, "n_completed_total": 339, "n_after_warmup": 315, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 34774.269216, "ttft_ms_p90": 64874.196589, "ttft_ms_p99": 70307.478291, "tpot_ms_p50": 90.2518300627451, "tpot_ms_p90": 92.71257150196078, "tpot_ms_p99": 101.1596845882353, "e2e_ms_p50": 57836.284489, "e2e_ms_p90": 80611.595233, "e2e_ms_p99": 85181.738441, "throughput_effective_rps": 1.3787875418878743, "throughput_ratio": 0.6893937709439372, "inflight_p50": 74, "inflight_p90": 125, "phase": "A", "cell": "A_r2.0_4096x256" } ]mooncake_producer [ { "rate_target": 0.5, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 401.356396668, "n_completed_total": 203, "n_after_warmup": 197, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 264.054176, "ttft_ms_p90": 453.351916, "ttft_ms_p99": 663.441612, "tpot_ms_p50": 10.750173619607843, "tpot_ms_p90": 16.46543019607843, "tpot_ms_p99": 19.549762741176473, "e2e_ms_p50": 3036.897016, "e2e_ms_p90": 4472.36076, "e2e_ms_p99": 5392.21106, "throughput_effective_rps": 0.5033774883386443, "throughput_ratio": 1.0067549766772885, "inflight_p50": 2, "inflight_p90": 5, "phase": "A", "cell": "A_r0.5_4096x256" }, { "rate_target": 1.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 232.645091396, "n_completed_total": 207, "n_after_warmup": 199, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 293.729297, "ttft_ms_p90": 615.19522, "ttft_ms_p99": 817.593711, "tpot_ms_p50": 25.506409952941176, "tpot_ms_p90": 39.46163431372549, "tpot_ms_p99": 48.344151764705884, "e2e_ms_p50": 6860.61264, "e2e_ms_p90": 10558.202004, "e2e_ms_p99": 12793.997244, "throughput_effective_rps": 0.8937991794575678, "throughput_ratio": 0.8937991794575678, "inflight_p50": 6, "inflight_p90": 14, "phase": "A", "cell": "A_r1.0_4096x256" }, { "rate_target": 2.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 232.801098162, "n_completed_total": 326, "n_after_warmup": 313, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 32682.299634, "ttft_ms_p90": 60520.025556, "ttft_ms_p99": 65848.06529, "tpot_ms_p50": 89.82475232156862, "tpot_ms_p90": 90.80036660392157, "tpot_ms_p99": 105.81740028627452, "e2e_ms_p50": 55363.619077, "e2e_ms_p90": 76987.277716, "e2e_ms_p99": 85493.952539, "throughput_effective_rps": 1.404840472430777, "throughput_ratio": 0.7024202362153885, "inflight_p50": 74, "inflight_p90": 117, "phase": "A", "cell": "A_r2.0_4096x256" } ]mooncake_both [ { "rate_target": 0.5, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 445.059962844, "n_completed_total": 200, "n_after_warmup": 196, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 256.440661, "ttft_ms_p90": 266.103771, "ttft_ms_p99": 461.167638, "tpot_ms_p50": 5.941321478431372, "tpot_ms_p90": 8.577892874509804, "tpot_ms_p99": 13.845412023529411, "e2e_ms_p50": 1807.769317, "e2e_ms_p90": 2514.381893, "e2e_ms_p99": 3815.785105, "throughput_effective_rps": 0.45051261145415944, "throughput_ratio": 0.9010252229083189, "inflight_p50": 1, "inflight_p90": 3, "phase": "A", "cell": "A_r0.5_4096x256" }, { "rate_target": 1.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 206.654679737, "n_completed_total": 203, "n_after_warmup": 186, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 264.012039, "ttft_ms_p90": 421.973366, "ttft_ms_p99": 614.736154, "tpot_ms_p50": 9.826510094117646, "tpot_ms_p90": 16.444674933333335, "tpot_ms_p99": 19.560320356862743, "e2e_ms_p50": 2785.772229, "e2e_ms_p90": 4532.58161, "e2e_ms_p99": 5253.24676, "throughput_effective_rps": 0.9458203600786452, "throughput_ratio": 0.9458203600786452, "inflight_p50": 3, "inflight_p90": 8, "phase": "A", "cell": "A_r1.0_4096x256" }, { "rate_target": 2.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 109.933145689, "n_completed_total": 237, "n_after_warmup": 208, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 316.799702, "ttft_ms_p90": 725.661939, "ttft_ms_p99": 945.498778, "tpot_ms_p50": 35.43492209411765, "tpot_ms_p90": 54.209878376470584, "tpot_ms_p99": 58.219066156862745, "e2e_ms_p50": 9571.144788, "e2e_ms_p90": 14359.084682, "e2e_ms_p99": 15374.620667, "throughput_effective_rps": 2.081391499946502, "throughput_ratio": 1.040695749973251, "inflight_p50": 23, "inflight_p90": 38, "phase": "A", "cell": "A_r2.0_4096x256" }, { "rate_target": 4.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 124.259570043, "n_completed_total": 343, "n_after_warmup": 307, "n_dropped": 0, "n_errors": 0, "ttft_ms_p50": 11665.573018, "ttft_ms_p90": 23444.380711, "ttft_ms_p99": 27466.05648, "tpot_ms_p50": 91.8016532509804, "tpot_ms_p90": 93.95965819607844, "tpot_ms_p99": 95.09967682352942, "e2e_ms_p50": 34656.343746, "e2e_ms_p90": 40580.01734, "e2e_ms_p99": 43193.129063, "throughput_effective_rps": 2.686864652864218, "throughput_ratio": 0.6717161632160545, "inflight_p50": 101, "inflight_p90": 130, "phase": "A", "cell": "A_r4.0_4096x256" }, { "rate_target": 8.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 80.65869788, "n_completed_total": 480, "n_after_warmup": 96, "n_dropped": 103, "n_errors": 202, "ttft_ms_p50": 22195.273494, "ttft_ms_p90": 33607.452699, "ttft_ms_p99": 34484.768556, "tpot_ms_p50": 88.98730769019608, "tpot_ms_p90": 89.86937862352941, "tpot_ms_p99": 92.71757069803921, "e2e_ms_p50": 44887.058146, "e2e_ms_p90": 56519.577831, "e2e_ms_p99": 57394.551538, "throughput_effective_rps": 1.3586437746565503, "throughput_ratio": 0.16983047183206879, "inflight_p50": 127, "inflight_p90": 165, "phase": "A", "cell": "A_r8.0_4096x256" }, { "rate_target": 16.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 60.028652326, "n_completed_total": 916, "n_after_warmup": 0, "n_dropped": 0, "n_errors": 916, "ttft_ms_p50": null, "ttft_ms_p90": null, "ttft_ms_p99": null, "tpot_ms_p50": null, "tpot_ms_p90": null, "tpot_ms_p99": null, "e2e_ms_p50": null, "e2e_ms_p90": null, "e2e_ms_p99": null, "throughput_effective_rps": 0.0, "throughput_ratio": 0.0, "inflight_p50": null, "inflight_p90": null, "phase": "A", "cell": "A_r16.0_4096x256" }, { "rate_target": 32.0, "input_tokens": 4096, "output_tokens": 256, "duration_actual_s": 60.011136933, "n_completed_total": 1815, "n_after_warmup": 0, "n_dropped": 0, "n_errors": 1815, "ttft_ms_p50": null, "ttft_ms_p90": null, "ttft_ms_p99": null, "tpot_ms_p50": null, "tpot_ms_p90": null, "tpot_ms_p99": null, "e2e_ms_p50": null, "e2e_ms_p90": null, "e2e_ms_p99": null, "throughput_effective_rps": 0.0, "throughput_ratio": 0.0, "inflight_p50": null, "inflight_p90": null, "phase": "A", "cell": "A_r32.0_4096x256" } ]