27856 lines
862 KiB
JSON
27856 lines
862 KiB
JSON
{
|
|
"config_results": [
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"loads": [
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 190.62253308296204,
|
|
"offered_request_rate": 0.1,
|
|
"request_metrics_sha256": "b45cd3a473a98bee1127b2315d0c0c59ded580275f6e6d10a3e5a52d85d24de4",
|
|
"request_rate_per_gpu": 0.025,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 11656.879780530655,
|
|
"request_id": 0,
|
|
"tpot_ms": 89.67953789863164,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 11729.945143691864,
|
|
"request_id": 1,
|
|
"tpot_ms": 89.99959080602144,
|
|
"ttft_ms": 299.9971113271407
|
|
},
|
|
{
|
|
"e2e_ms": 11712.730146443742,
|
|
"request_id": 2,
|
|
"tpot_ms": 90.00056214193414,
|
|
"ttft_ms": 282.65875441810806
|
|
},
|
|
{
|
|
"e2e_ms": 11785.795509604945,
|
|
"request_id": 3,
|
|
"tpot_ms": 89.99959080602139,
|
|
"ttft_ms": 355.8474772402285
|
|
},
|
|
{
|
|
"e2e_ms": 11770.95360336775,
|
|
"request_id": 4,
|
|
"tpot_ms": 90.01924789792568,
|
|
"ttft_ms": 338.50912033118874
|
|
},
|
|
{
|
|
"e2e_ms": 11756.111697130556,
|
|
"request_id": 5,
|
|
"tpot_ms": 90.0192478979257,
|
|
"ttft_ms": 323.66721409399446
|
|
},
|
|
{
|
|
"e2e_ms": 11741.269790893028,
|
|
"request_id": 6,
|
|
"tpot_ms": 90.01924789792305,
|
|
"ttft_ms": 308.8253078568002
|
|
},
|
|
{
|
|
"e2e_ms": 11726.427884655508,
|
|
"request_id": 7,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 293.98340161927194
|
|
},
|
|
{
|
|
"e2e_ms": 11709.212887407048,
|
|
"request_id": 8,
|
|
"tpot_ms": 90.00056214193147,
|
|
"ttft_ms": 279.1414953817508
|
|
},
|
|
{
|
|
"e2e_ms": 11782.278250567928,
|
|
"request_id": 9,
|
|
"tpot_ms": 89.99959080601886,
|
|
"ttft_ms": 352.3302182035337
|
|
},
|
|
{
|
|
"e2e_ms": 11767.436344330406,
|
|
"request_id": 10,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 334.9918612941707
|
|
},
|
|
{
|
|
"e2e_ms": 11752.594438092885,
|
|
"request_id": 11,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 320.14995505664956
|
|
},
|
|
{
|
|
"e2e_ms": 11737.752531855705,
|
|
"request_id": 12,
|
|
"tpot_ms": 90.01924789792581,
|
|
"ttft_ms": 305.30804881912843
|
|
},
|
|
{
|
|
"e2e_ms": 11722.91062561817,
|
|
"request_id": 13,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 290.46614258194836
|
|
},
|
|
{
|
|
"e2e_ms": 11705.69562836971,
|
|
"request_id": 14,
|
|
"tpot_ms": 90.00056214193148,
|
|
"ttft_ms": 275.624236344413
|
|
},
|
|
{
|
|
"e2e_ms": 11778.76099153056,
|
|
"request_id": 15,
|
|
"tpot_ms": 89.99959080601862,
|
|
"ttft_ms": 348.8129591661959
|
|
},
|
|
{
|
|
"e2e_ms": 11763.919085293026,
|
|
"request_id": 16,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 331.4746022568045
|
|
},
|
|
{
|
|
"e2e_ms": 11749.07717905549,
|
|
"request_id": 17,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 316.63269601926913
|
|
},
|
|
{
|
|
"e2e_ms": 11734.235272817954,
|
|
"request_id": 18,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 301.7907897817338
|
|
},
|
|
{
|
|
"e2e_ms": 11719.39336658042,
|
|
"request_id": 19,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 286.94888354419845
|
|
},
|
|
{
|
|
"e2e_ms": 11702.178369331961,
|
|
"request_id": 20,
|
|
"tpot_ms": 90.00056214193148,
|
|
"ttft_ms": 272.1069773066631
|
|
},
|
|
{
|
|
"e2e_ms": 11775.24373249281,
|
|
"request_id": 21,
|
|
"tpot_ms": 89.99959080601862,
|
|
"ttft_ms": 345.295700128446
|
|
},
|
|
{
|
|
"e2e_ms": 11760.401826255276,
|
|
"request_id": 22,
|
|
"tpot_ms": 90.01924789792301,
|
|
"ttft_ms": 327.95734321905456
|
|
},
|
|
{
|
|
"e2e_ms": 11745.55992001774,
|
|
"request_id": 23,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 313.1154369815192
|
|
},
|
|
{
|
|
"e2e_ms": 11730.718013780204,
|
|
"request_id": 24,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 298.2735307439839
|
|
},
|
|
{
|
|
"e2e_ms": 11713.50301653041,
|
|
"request_id": 25,
|
|
"tpot_ms": 90.00056214192095,
|
|
"ttft_ms": 283.43162450644854
|
|
},
|
|
{
|
|
"e2e_ms": 11786.568379689925,
|
|
"request_id": 26,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 356.6203473268956
|
|
},
|
|
{
|
|
"e2e_ms": 11771.726473451054,
|
|
"request_id": 27,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 339.28199041616836
|
|
},
|
|
{
|
|
"e2e_ms": 11756.884567212182,
|
|
"request_id": 28,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 324.4400841772972
|
|
},
|
|
{
|
|
"e2e_ms": 11742.042660973311,
|
|
"request_id": 29,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 309.59817793842603
|
|
},
|
|
{
|
|
"e2e_ms": 11727.20075473444,
|
|
"request_id": 30,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 294.7562716995549
|
|
},
|
|
{
|
|
"e2e_ms": 11709.985757484617,
|
|
"request_id": 31,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 279.9143654606837
|
|
},
|
|
{
|
|
"e2e_ms": 11783.051120644132,
|
|
"request_id": 32,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 353.1030882811024
|
|
},
|
|
{
|
|
"e2e_ms": 11768.20921440526,
|
|
"request_id": 33,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 335.7647313703751
|
|
},
|
|
{
|
|
"e2e_ms": 11753.36730816639,
|
|
"request_id": 34,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 320.92282513150394
|
|
},
|
|
{
|
|
"e2e_ms": 11738.525401927518,
|
|
"request_id": 35,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 306.0809188926328
|
|
},
|
|
{
|
|
"e2e_ms": 11723.683495688647,
|
|
"request_id": 36,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 291.2390126537616
|
|
},
|
|
{
|
|
"e2e_ms": 11706.468498438824,
|
|
"request_id": 37,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 276.39710641489046
|
|
},
|
|
{
|
|
"e2e_ms": 11779.533861598338,
|
|
"request_id": 38,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 349.5858292353091
|
|
},
|
|
{
|
|
"e2e_ms": 11764.691955359467,
|
|
"request_id": 39,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 332.24747232458185
|
|
},
|
|
{
|
|
"e2e_ms": 11749.850049120596,
|
|
"request_id": 40,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 317.4055660857107
|
|
},
|
|
{
|
|
"e2e_ms": 11735.008142881725,
|
|
"request_id": 41,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 302.5636598468395
|
|
},
|
|
{
|
|
"e2e_ms": 11720.166236642854,
|
|
"request_id": 42,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 287.72175360796837
|
|
},
|
|
{
|
|
"e2e_ms": 11702.95123939303,
|
|
"request_id": 43,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 272.8798473690972
|
|
},
|
|
{
|
|
"e2e_ms": 11776.016602552545,
|
|
"request_id": 44,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 346.06857018951587
|
|
},
|
|
{
|
|
"e2e_ms": 11761.174696313674,
|
|
"request_id": 45,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 328.7302132787886
|
|
},
|
|
{
|
|
"e2e_ms": 11746.332790074803,
|
|
"request_id": 46,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 313.88830703991744
|
|
},
|
|
{
|
|
"e2e_ms": 11731.490883835932,
|
|
"request_id": 47,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 299.0464008010463
|
|
},
|
|
{
|
|
"e2e_ms": 11716.64897759706,
|
|
"request_id": 48,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 284.2044945621751
|
|
},
|
|
{
|
|
"e2e_ms": 11699.433980347238,
|
|
"request_id": 49,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 269.36258832330395
|
|
},
|
|
{
|
|
"e2e_ms": 11772.499343506752,
|
|
"request_id": 50,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 342.5513111437226
|
|
},
|
|
{
|
|
"e2e_ms": 11757.657437270042,
|
|
"request_id": 51,
|
|
"tpot_ms": 90.0192478979295,
|
|
"ttft_ms": 325.21295423299534
|
|
},
|
|
{
|
|
"e2e_ms": 11742.815531033444,
|
|
"request_id": 52,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 310.3710479967958
|
|
},
|
|
{
|
|
"e2e_ms": 11727.973624796847,
|
|
"request_id": 53,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 295.5291417601984
|
|
},
|
|
{
|
|
"e2e_ms": 11710.758627549352,
|
|
"request_id": 54,
|
|
"tpot_ms": 90.00056214193505,
|
|
"ttft_ms": 280.687235523601
|
|
},
|
|
{
|
|
"e2e_ms": 11783.823990711198,
|
|
"request_id": 55,
|
|
"tpot_ms": 89.99959080602287,
|
|
"ttft_ms": 353.8759583462934
|
|
},
|
|
{
|
|
"e2e_ms": 11768.982084474601,
|
|
"request_id": 56,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 336.5376014379535
|
|
},
|
|
{
|
|
"e2e_ms": 11754.140178238004,
|
|
"request_id": 57,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 321.6956952013561
|
|
},
|
|
{
|
|
"e2e_ms": 11739.298272001406,
|
|
"request_id": 58,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 306.8537889647587
|
|
},
|
|
{
|
|
"e2e_ms": 11724.456365764809,
|
|
"request_id": 59,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 292.01188272816125
|
|
},
|
|
{
|
|
"e2e_ms": 11707.241368517316,
|
|
"request_id": 60,
|
|
"tpot_ms": 90.00056214193505,
|
|
"ttft_ms": 277.16997649156383
|
|
},
|
|
{
|
|
"e2e_ms": 11780.30673167916,
|
|
"request_id": 61,
|
|
"tpot_ms": 89.99959080602287,
|
|
"ttft_ms": 350.3586993142562
|
|
},
|
|
{
|
|
"e2e_ms": 11765.464825442563,
|
|
"request_id": 62,
|
|
"tpot_ms": 90.01924789792635,
|
|
"ttft_ms": 333.0203424059164
|
|
},
|
|
{
|
|
"e2e_ms": 11530.943387303068,
|
|
"request_id": 63,
|
|
"tpot_ms": 88.28948780420275,
|
|
"ttft_ms": 318.17843616931896
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "37e166453683700c9df7abffa278431119205a29c83e92211ffa69d90e1dca0e"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 27.596914052963257,
|
|
"offered_request_rate": 0.2,
|
|
"request_metrics_sha256": "7930ffb53e440b9eefa724912c777842a9c33cefdb46d9014e1c5003223221d2",
|
|
"request_rate_per_gpu": 0.05,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 11975.67900227787,
|
|
"request_id": 0,
|
|
"tpot_ms": 92.18976799112939,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 12131.199338988026,
|
|
"request_id": 1,
|
|
"tpot_ms": 93.2890801728389,
|
|
"ttft_ms": 283.4861570374869
|
|
},
|
|
{
|
|
"e2e_ms": 12196.411822684866,
|
|
"request_id": 2,
|
|
"tpot_ms": 93.31228634935276,
|
|
"ttft_ms": 345.75145631706584
|
|
},
|
|
{
|
|
"e2e_ms": 12171.220586311463,
|
|
"request_id": 3,
|
|
"tpot_ms": 93.31547315047263,
|
|
"ttft_ms": 320.15549620144054
|
|
},
|
|
{
|
|
"e2e_ms": 12145.933482881133,
|
|
"request_id": 4,
|
|
"tpot_ms": 93.31640326087744,
|
|
"ttft_ms": 294.75026874969853
|
|
},
|
|
{
|
|
"e2e_ms": 12120.646379450847,
|
|
"request_id": 5,
|
|
"tpot_ms": 93.31564840216181,
|
|
"ttft_ms": 269.5590323762964
|
|
},
|
|
{
|
|
"e2e_ms": 12185.858863147729,
|
|
"request_id": 6,
|
|
"tpot_ms": 93.31471829175732,
|
|
"ttft_ms": 334.88964009454844
|
|
},
|
|
{
|
|
"e2e_ms": 12160.571759717443,
|
|
"request_id": 7,
|
|
"tpot_ms": 93.31471829175764,
|
|
"ttft_ms": 309.6025366642223
|
|
},
|
|
{
|
|
"e2e_ms": 12135.284656287155,
|
|
"request_id": 8,
|
|
"tpot_ms": 93.31564840216245,
|
|
"ttft_ms": 284.19730921252295
|
|
},
|
|
{
|
|
"e2e_ms": 12200.497139984038,
|
|
"request_id": 9,
|
|
"tpot_ms": 93.31471829175764,
|
|
"ttft_ms": 349.52791693081764
|
|
},
|
|
{
|
|
"e2e_ms": 12175.305903610677,
|
|
"request_id": 10,
|
|
"tpot_ms": 93.3154731504736,
|
|
"ttft_ms": 324.2408135005306
|
|
},
|
|
{
|
|
"e2e_ms": 12150.018800180305,
|
|
"request_id": 11,
|
|
"tpot_ms": 93.31640326087775,
|
|
"ttft_ms": 298.8355860488312
|
|
},
|
|
{
|
|
"e2e_ms": 12124.731696749934,
|
|
"request_id": 12,
|
|
"tpot_ms": 93.31564840216113,
|
|
"ttft_ms": 273.6443496754717
|
|
},
|
|
{
|
|
"e2e_ms": 12189.94418044673,
|
|
"request_id": 13,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 338.97495739375927
|
|
},
|
|
{
|
|
"e2e_ms": 12164.657077016358,
|
|
"request_id": 14,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 313.68785396338694
|
|
},
|
|
{
|
|
"e2e_ms": 12139.369973585985,
|
|
"request_id": 15,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 288.2826265116023
|
|
},
|
|
{
|
|
"e2e_ms": 12204.582457282782,
|
|
"request_id": 16,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 353.6132342298117
|
|
},
|
|
{
|
|
"e2e_ms": 12179.391220909338,
|
|
"request_id": 17,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 328.3261307994394
|
|
},
|
|
{
|
|
"e2e_ms": 12154.104117478966,
|
|
"request_id": 18,
|
|
"tpot_ms": 93.31640326087647,
|
|
"ttft_ms": 302.92090334765476
|
|
},
|
|
{
|
|
"e2e_ms": 12128.817014048593,
|
|
"request_id": 19,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 277.72966697421
|
|
},
|
|
{
|
|
"e2e_ms": 12194.02949774539,
|
|
"request_id": 20,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 343.0602746924194
|
|
},
|
|
{
|
|
"e2e_ms": 12168.742394315017,
|
|
"request_id": 21,
|
|
"tpot_ms": 93.31471829175567,
|
|
"ttft_ms": 317.7731712620471
|
|
},
|
|
{
|
|
"e2e_ms": 12143.455290884645,
|
|
"request_id": 22,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 292.36794381026243
|
|
},
|
|
{
|
|
"e2e_ms": 12208.667774581443,
|
|
"request_id": 23,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 357.69855152847185
|
|
},
|
|
{
|
|
"e2e_ms": 12183.476538207999,
|
|
"request_id": 24,
|
|
"tpot_ms": 93.31547315047165,
|
|
"ttft_ms": 332.4114480980995
|
|
},
|
|
{
|
|
"e2e_ms": 12158.189434777625,
|
|
"request_id": 25,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 307.0062206463149
|
|
},
|
|
{
|
|
"e2e_ms": 12132.902331347253,
|
|
"request_id": 26,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 281.8149842728701
|
|
},
|
|
{
|
|
"e2e_ms": 12198.11481504405,
|
|
"request_id": 27,
|
|
"tpot_ms": 93.31471829175567,
|
|
"ttft_ms": 347.1455919910795
|
|
},
|
|
{
|
|
"e2e_ms": 12172.923578670605,
|
|
"request_id": 28,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 321.8584885607072
|
|
},
|
|
{
|
|
"e2e_ms": 12147.636475240233,
|
|
"request_id": 29,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 296.45326110892256
|
|
},
|
|
{
|
|
"e2e_ms": 12122.349371809862,
|
|
"request_id": 30,
|
|
"tpot_ms": 93.3156484021605,
|
|
"ttft_ms": 271.2620247354778
|
|
},
|
|
{
|
|
"e2e_ms": 12187.561855506658,
|
|
"request_id": 31,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 336.5926324536872
|
|
},
|
|
{
|
|
"e2e_ms": 12162.274752076286,
|
|
"request_id": 32,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 311.3055290233149
|
|
},
|
|
{
|
|
"e2e_ms": 12136.987648645914,
|
|
"request_id": 33,
|
|
"tpot_ms": 93.3156484021605,
|
|
"ttft_ms": 285.90030157153024
|
|
},
|
|
{
|
|
"e2e_ms": 12202.20013234271,
|
|
"request_id": 34,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 351.23090928973966
|
|
},
|
|
{
|
|
"e2e_ms": 12177.008895969266,
|
|
"request_id": 35,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 325.94380585936733
|
|
},
|
|
{
|
|
"e2e_ms": 12151.721792538894,
|
|
"request_id": 36,
|
|
"tpot_ms": 93.31640326087647,
|
|
"ttft_ms": 300.5385784075827
|
|
},
|
|
{
|
|
"e2e_ms": 12126.43468910852,
|
|
"request_id": 37,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 275.3473420341379
|
|
},
|
|
{
|
|
"e2e_ms": 12191.647172805318,
|
|
"request_id": 38,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 340.67794975234733
|
|
},
|
|
{
|
|
"e2e_ms": 12166.360069374947,
|
|
"request_id": 39,
|
|
"tpot_ms": 93.3147182917557,
|
|
"ttft_ms": 315.390846321975
|
|
},
|
|
{
|
|
"e2e_ms": 12141.072965944573,
|
|
"request_id": 40,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 289.98561887019036
|
|
},
|
|
{
|
|
"e2e_ms": 12206.285449641371,
|
|
"request_id": 41,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 355.3162265883998
|
|
},
|
|
{
|
|
"e2e_ms": 12181.094213267927,
|
|
"request_id": 42,
|
|
"tpot_ms": 93.31547315047165,
|
|
"ttft_ms": 330.02912315802746
|
|
},
|
|
{
|
|
"e2e_ms": 12155.807109837553,
|
|
"request_id": 43,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 304.6238957062428
|
|
},
|
|
{
|
|
"e2e_ms": 12130.520006407181,
|
|
"request_id": 44,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 279.43265933279804
|
|
},
|
|
{
|
|
"e2e_ms": 12195.73249010398,
|
|
"request_id": 45,
|
|
"tpot_ms": 93.3147182917557,
|
|
"ttft_ms": 344.76326705100746
|
|
},
|
|
{
|
|
"e2e_ms": 12170.445386673606,
|
|
"request_id": 46,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 319.47616362063513
|
|
},
|
|
{
|
|
"e2e_ms": 12145.158283243234,
|
|
"request_id": 47,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 294.0709361688505
|
|
},
|
|
{
|
|
"e2e_ms": 12210.37076694003,
|
|
"request_id": 48,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 359.4015438870599
|
|
},
|
|
{
|
|
"e2e_ms": 12185.179530566586,
|
|
"request_id": 49,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 334.1144404566876
|
|
},
|
|
{
|
|
"e2e_ms": 12159.892427136241,
|
|
"request_id": 50,
|
|
"tpot_ms": 93.31640326087668,
|
|
"ttft_ms": 308.70921300490295
|
|
},
|
|
{
|
|
"e2e_ms": 12134.605323705899,
|
|
"request_id": 51,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 283.51797663145817
|
|
},
|
|
{
|
|
"e2e_ms": 12199.817807402724,
|
|
"request_id": 52,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 348.848584349696
|
|
},
|
|
{
|
|
"e2e_ms": 12174.626571029308,
|
|
"request_id": 53,
|
|
"tpot_ms": 93.3154731504721,
|
|
"ttft_ms": 323.5614809193521
|
|
},
|
|
{
|
|
"e2e_ms": 12149.339467598964,
|
|
"request_id": 54,
|
|
"tpot_ms": 93.31640326087691,
|
|
"ttft_ms": 298.1562534675959
|
|
},
|
|
{
|
|
"e2e_ms": 12124.05236416862,
|
|
"request_id": 55,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 272.96501709417953
|
|
},
|
|
{
|
|
"e2e_ms": 12189.264847865445,
|
|
"request_id": 56,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 338.2956248124174
|
|
},
|
|
{
|
|
"e2e_ms": 12163.977744435102,
|
|
"request_id": 57,
|
|
"tpot_ms": 93.31471829175614,
|
|
"ttft_ms": 313.00852138207347
|
|
},
|
|
{
|
|
"e2e_ms": 12138.690641004758,
|
|
"request_id": 58,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 287.60329393031725
|
|
},
|
|
{
|
|
"e2e_ms": 12203.903124701583,
|
|
"request_id": 59,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 352.9339016485551
|
|
},
|
|
{
|
|
"e2e_ms": 12178.711888328166,
|
|
"request_id": 60,
|
|
"tpot_ms": 93.31547315047209,
|
|
"ttft_ms": 327.6467982182112
|
|
},
|
|
{
|
|
"e2e_ms": 12153.424784897823,
|
|
"request_id": 61,
|
|
"tpot_ms": 93.31640326087691,
|
|
"ttft_ms": 302.24157076645497
|
|
},
|
|
{
|
|
"e2e_ms": 11949.074413391485,
|
|
"request_id": 62,
|
|
"tpot_ms": 91.90570140943657,
|
|
"ttft_ms": 277.0503343930386
|
|
},
|
|
{
|
|
"e2e_ms": 11702.728382593137,
|
|
"request_id": 63,
|
|
"tpot_ms": 89.45155464946347,
|
|
"ttft_ms": 342.38094211127645
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "e6cebfc858981b2ae4632d2c75f5e8c1f83280e7306351e15c7cd611d2664d80"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 22.99045443534851,
|
|
"offered_request_rate": 0.4,
|
|
"request_metrics_sha256": "e62567aa33ec429331fcd9f24053a0c79db61ef288f7994e6488ae0f0fa79b7e",
|
|
"request_rate_per_gpu": 0.1,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12394.3405299904,
|
|
"request_id": 0,
|
|
"tpot_ms": 95.48631545343278,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 12520.094438712134,
|
|
"request_id": 1,
|
|
"tpot_ms": 96.04733720338449,
|
|
"ttft_ms": 322.08261388230494
|
|
},
|
|
{
|
|
"e2e_ms": 12556.880443424032,
|
|
"request_id": 2,
|
|
"tpot_ms": 96.10807445129032,
|
|
"ttft_ms": 351.1549881101628
|
|
},
|
|
{
|
|
"e2e_ms": 12502.892311945501,
|
|
"request_id": 3,
|
|
"tpot_ms": 96.14618123456218,
|
|
"ttft_ms": 292.32729515610424
|
|
},
|
|
{
|
|
"e2e_ms": 12539.549613984442,
|
|
"request_id": 4,
|
|
"tpot_ms": 96.30977229626299,
|
|
"ttft_ms": 308.2085323590427
|
|
},
|
|
{
|
|
"e2e_ms": 12576.335618696337,
|
|
"request_id": 5,
|
|
"tpot_ms": 96.32399459689755,
|
|
"ttft_ms": 343.188304890349
|
|
},
|
|
{
|
|
"e2e_ms": 12522.347487217807,
|
|
"request_id": 6,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 289.200173411821
|
|
},
|
|
{
|
|
"e2e_ms": 12559.133491929704,
|
|
"request_id": 7,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 325.98617812371697
|
|
},
|
|
{
|
|
"e2e_ms": 12505.145360451173,
|
|
"request_id": 8,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 271.99804664518723
|
|
},
|
|
{
|
|
"e2e_ms": 12541.802662490163,
|
|
"request_id": 9,
|
|
"tpot_ms": 96.32399459689792,
|
|
"ttft_ms": 308.6553486841268
|
|
},
|
|
{
|
|
"e2e_ms": 12578.588667202104,
|
|
"request_id": 10,
|
|
"tpot_ms": 96.32399459689827,
|
|
"ttft_ms": 345.4413533960228
|
|
},
|
|
{
|
|
"e2e_ms": 12524.60053572362,
|
|
"request_id": 11,
|
|
"tpot_ms": 96.32399459689864,
|
|
"ttft_ms": 291.453221917493
|
|
},
|
|
{
|
|
"e2e_ms": 12561.386540435564,
|
|
"request_id": 12,
|
|
"tpot_ms": 96.32399459689903,
|
|
"ttft_ms": 328.239226629389
|
|
},
|
|
{
|
|
"e2e_ms": 12507.39840895708,
|
|
"request_id": 13,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 274.2510951508592
|
|
},
|
|
{
|
|
"e2e_ms": 12544.05571099607,
|
|
"request_id": 14,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 310.90839718984853
|
|
},
|
|
{
|
|
"e2e_ms": 12580.841715708011,
|
|
"request_id": 15,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 347.6944019017907
|
|
},
|
|
{
|
|
"e2e_ms": 12526.853584229528,
|
|
"request_id": 16,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 293.7062704233071
|
|
},
|
|
{
|
|
"e2e_ms": 12563.63958894147,
|
|
"request_id": 17,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 330.49227513524926
|
|
},
|
|
{
|
|
"e2e_ms": 12509.651457462987,
|
|
"request_id": 18,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 276.5041436567657
|
|
},
|
|
{
|
|
"e2e_ms": 12546.308759501975,
|
|
"request_id": 19,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 313.161445695755
|
|
},
|
|
{
|
|
"e2e_ms": 12583.094764213918,
|
|
"request_id": 20,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 349.94745040769715
|
|
},
|
|
{
|
|
"e2e_ms": 12529.106632735506,
|
|
"request_id": 21,
|
|
"tpot_ms": 96.32399459689994,
|
|
"ttft_ms": 295.95931892921357
|
|
},
|
|
{
|
|
"e2e_ms": 12565.892637447447,
|
|
"request_id": 22,
|
|
"tpot_ms": 96.32399459689992,
|
|
"ttft_ms": 332.7453236411557
|
|
},
|
|
{
|
|
"e2e_ms": 12511.904505968956,
|
|
"request_id": 23,
|
|
"tpot_ms": 96.32399459689988,
|
|
"ttft_ms": 278.75719216267214
|
|
},
|
|
{
|
|
"e2e_ms": 12548.561808007931,
|
|
"request_id": 24,
|
|
"tpot_ms": 96.32399459689977,
|
|
"ttft_ms": 315.41449420166146
|
|
},
|
|
{
|
|
"e2e_ms": 12585.347812719874,
|
|
"request_id": 25,
|
|
"tpot_ms": 96.32399459689977,
|
|
"ttft_ms": 352.2004989136036
|
|
},
|
|
{
|
|
"e2e_ms": 12531.359681241383,
|
|
"request_id": 26,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 298.2123674351982
|
|
},
|
|
{
|
|
"e2e_ms": 12568.145685953326,
|
|
"request_id": 27,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 334.99837214714034
|
|
},
|
|
{
|
|
"e2e_ms": 12514.157554474836,
|
|
"request_id": 28,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 281.01024066864966
|
|
},
|
|
{
|
|
"e2e_ms": 12550.81485651381,
|
|
"request_id": 29,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 317.66754270762476
|
|
},
|
|
{
|
|
"e2e_ms": 12587.600861225752,
|
|
"request_id": 30,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 354.4535474195669
|
|
},
|
|
{
|
|
"e2e_ms": 12533.612729747261,
|
|
"request_id": 31,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 300.46541594107623
|
|
},
|
|
{
|
|
"e2e_ms": 12570.398734459204,
|
|
"request_id": 32,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 337.2514206530184
|
|
},
|
|
{
|
|
"e2e_ms": 12516.410602980714,
|
|
"request_id": 33,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 283.2632891745277
|
|
},
|
|
{
|
|
"e2e_ms": 12553.067905019689,
|
|
"request_id": 34,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 319.9205912135028
|
|
},
|
|
{
|
|
"e2e_ms": 12589.85390973163,
|
|
"request_id": 35,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 356.70659592544496
|
|
},
|
|
{
|
|
"e2e_ms": 12535.865778253139,
|
|
"request_id": 36,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 302.7184644469543
|
|
},
|
|
{
|
|
"e2e_ms": 12572.651782965082,
|
|
"request_id": 37,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 339.5044691588964
|
|
},
|
|
{
|
|
"e2e_ms": 12518.663651486591,
|
|
"request_id": 38,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 285.51633768040574
|
|
},
|
|
{
|
|
"e2e_ms": 12555.320953525566,
|
|
"request_id": 39,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 322.17363971938084
|
|
},
|
|
{
|
|
"e2e_ms": 12770.519034100516,
|
|
"request_id": 40,
|
|
"tpot_ms": 97.72881409188341,
|
|
"ttft_ms": 358.959644431323
|
|
},
|
|
{
|
|
"e2e_ms": 12538.070601439798,
|
|
"request_id": 41,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 304.9715129528323
|
|
},
|
|
{
|
|
"e2e_ms": 12574.856606151741,
|
|
"request_id": 42,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 341.75751766477447
|
|
},
|
|
{
|
|
"e2e_ms": 12520.86847467325,
|
|
"request_id": 43,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 287.7693861862838
|
|
},
|
|
{
|
|
"e2e_ms": 12557.654479385192,
|
|
"request_id": 44,
|
|
"tpot_ms": 96.32462827684986,
|
|
"ttft_ms": 324.4266882252589
|
|
},
|
|
{
|
|
"e2e_ms": 12503.666347906701,
|
|
"request_id": 45,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 270.51903410051636
|
|
},
|
|
{
|
|
"e2e_ms": 12540.323649945676,
|
|
"request_id": 46,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 307.17633613949147
|
|
},
|
|
{
|
|
"e2e_ms": 12577.109654657448,
|
|
"request_id": 47,
|
|
"tpot_ms": 96.32399459689775,
|
|
"ttft_ms": 343.9623408514336
|
|
},
|
|
{
|
|
"e2e_ms": 12523.121523178786,
|
|
"request_id": 48,
|
|
"tpot_ms": 96.3239945968964,
|
|
"ttft_ms": 289.97420937294294
|
|
},
|
|
{
|
|
"e2e_ms": 12559.907527890544,
|
|
"request_id": 49,
|
|
"tpot_ms": 96.32399459689495,
|
|
"ttft_ms": 326.7602140848851
|
|
},
|
|
{
|
|
"e2e_ms": 12505.919396411884,
|
|
"request_id": 50,
|
|
"tpot_ms": 96.32399459689363,
|
|
"ttft_ms": 272.7720826063944
|
|
},
|
|
{
|
|
"e2e_ms": 12542.576698450688,
|
|
"request_id": 51,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 309.4293846453695
|
|
},
|
|
{
|
|
"e2e_ms": 12579.362703162446,
|
|
"request_id": 52,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 346.2153893571269
|
|
},
|
|
{
|
|
"e2e_ms": 12525.374571683784,
|
|
"request_id": 53,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 292.2272578784657
|
|
},
|
|
{
|
|
"e2e_ms": 12562.160576395541,
|
|
"request_id": 54,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 329.0132625902231
|
|
},
|
|
{
|
|
"e2e_ms": 12508.17244491688,
|
|
"request_id": 55,
|
|
"tpot_ms": 96.32399459689226,
|
|
"ttft_ms": 275.0251311115619
|
|
},
|
|
{
|
|
"e2e_ms": 12544.829746955686,
|
|
"request_id": 56,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 311.6824331503665
|
|
},
|
|
{
|
|
"e2e_ms": 12581.615751667443,
|
|
"request_id": 57,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 348.4684378621239
|
|
},
|
|
{
|
|
"e2e_ms": 12527.627620188781,
|
|
"request_id": 58,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 294.4803063834627
|
|
},
|
|
{
|
|
"e2e_ms": 12564.413624900539,
|
|
"request_id": 59,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 331.2663110952201
|
|
},
|
|
{
|
|
"e2e_ms": 12314.305135768365,
|
|
"request_id": 60,
|
|
"tpot_ms": 94.77973981221895,
|
|
"ttft_ms": 277.2781796165589
|
|
},
|
|
{
|
|
"e2e_ms": 12170.36562563149,
|
|
"request_id": 61,
|
|
"tpot_ms": 93.35771766910335,
|
|
"ttft_ms": 313.93548165536345
|
|
},
|
|
{
|
|
"e2e_ms": 12023.933572014499,
|
|
"request_id": 62,
|
|
"tpot_ms": 91.91505579249903,
|
|
"ttft_ms": 350.72148636712086
|
|
},
|
|
{
|
|
"e2e_ms": 11724.699298497455,
|
|
"request_id": 63,
|
|
"tpot_ms": 89.98398380794484,
|
|
"ttft_ms": 296.73335488845964
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "021442528b6c4067d63827496b66b6f530adbe9f4b29732b1aa7aab2112ced9f"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 23.932271718978882,
|
|
"offered_request_rate": 0.8,
|
|
"request_metrics_sha256": "b39b2c6a1c6236e30b246303e0b4d66b80a2515899d8c815efc015ef3f70db54",
|
|
"request_rate_per_gpu": 0.2,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 14028.106647728775,
|
|
"request_id": 0,
|
|
"tpot_ms": 108.35061559310502,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 14267.352384844213,
|
|
"request_id": 1,
|
|
"tpot_ms": 109.65331923259451,
|
|
"ttft_ms": 341.3808423047111
|
|
},
|
|
{
|
|
"e2e_ms": 14406.16288598222,
|
|
"request_id": 2,
|
|
"tpot_ms": 110.64064370419715,
|
|
"ttft_ms": 354.8011355491818
|
|
},
|
|
{
|
|
"e2e_ms": 14444.508775243683,
|
|
"request_id": 3,
|
|
"tpot_ms": 111.53949237897281,
|
|
"ttft_ms": 278.99324311413756
|
|
},
|
|
{
|
|
"e2e_ms": 14585.312621821135,
|
|
"request_id": 4,
|
|
"tpot_ms": 112.58765687327464,
|
|
"ttft_ms": 286.6801989152554
|
|
},
|
|
{
|
|
"e2e_ms": 14727.157204966976,
|
|
"request_id": 5,
|
|
"tpot_ms": 113.57630070139629,
|
|
"ttft_ms": 302.9670158896467
|
|
},
|
|
{
|
|
"e2e_ms": 14871.045643448613,
|
|
"request_id": 6,
|
|
"tpot_ms": 114.43037546464966,
|
|
"ttft_ms": 338.38795943810675
|
|
},
|
|
{
|
|
"e2e_ms": 14913.487485283544,
|
|
"request_id": 7,
|
|
"tpot_ms": 115.12950641852738,
|
|
"ttft_ms": 292.0401701305657
|
|
},
|
|
{
|
|
"e2e_ms": 15059.457396901958,
|
|
"request_id": 8,
|
|
"tpot_ms": 115.80285438725795,
|
|
"ttft_ms": 352.4948897201998
|
|
},
|
|
{
|
|
"e2e_ms": 15271.481983406971,
|
|
"request_id": 9,
|
|
"tpot_ms": 117.58757541921905,
|
|
"ttft_ms": 337.85990516615175
|
|
},
|
|
{
|
|
"e2e_ms": 15316.138289012584,
|
|
"request_id": 10,
|
|
"tpot_ms": 117.88145073243837,
|
|
"ttft_ms": 345.1940459929119
|
|
},
|
|
{
|
|
"e2e_ms": 15090.835365768904,
|
|
"request_id": 11,
|
|
"tpot_ms": 116.63565919716638,
|
|
"ttft_ms": 278.1066477287748
|
|
},
|
|
{
|
|
"e2e_ms": 15302.859952273917,
|
|
"request_id": 12,
|
|
"tpot_ms": 118.0348305674148,
|
|
"ttft_ms": 312.43647021223705
|
|
},
|
|
{
|
|
"e2e_ms": 15347.54563377864,
|
|
"request_id": 13,
|
|
"tpot_ms": 118.10044582475892,
|
|
"ttft_ms": 348.7890140342564
|
|
},
|
|
{
|
|
"e2e_ms": 15289.743982068245,
|
|
"request_id": 14,
|
|
"tpot_ms": 118.15013414242924,
|
|
"ttft_ms": 284.6769459797329
|
|
},
|
|
{
|
|
"e2e_ms": 15334.400287673823,
|
|
"request_id": 15,
|
|
"tpot_ms": 118.19982246009941,
|
|
"ttft_ms": 323.022835241197
|
|
},
|
|
{
|
|
"e2e_ms": 15380.492763872971,
|
|
"request_id": 16,
|
|
"tpot_ms": 118.2528557378034,
|
|
"ttft_ms": 362.38008517193873
|
|
},
|
|
{
|
|
"e2e_ms": 15322.691112162565,
|
|
"request_id": 17,
|
|
"tpot_ms": 118.27848723653601,
|
|
"ttft_ms": 301.32323312249196
|
|
},
|
|
{
|
|
"e2e_ms": 15367.376793667254,
|
|
"request_id": 18,
|
|
"tpot_ms": 118.29615526543176,
|
|
"ttft_ms": 343.765074957421
|
|
},
|
|
{
|
|
"e2e_ms": 15309.575141956848,
|
|
"request_id": 19,
|
|
"tpot_ms": 118.30562851819849,
|
|
"ttft_ms": 284.7603201456401
|
|
},
|
|
{
|
|
"e2e_ms": 15354.260823461536,
|
|
"request_id": 20,
|
|
"tpot_ms": 118.30690699483614,
|
|
"ttft_ms": 329.2836351173456
|
|
},
|
|
{
|
|
"e2e_ms": 15296.45917175113,
|
|
"request_id": 21,
|
|
"tpot_ms": 118.3069069948359,
|
|
"ttft_ms": 271.4819834069715
|
|
},
|
|
{
|
|
"e2e_ms": 15341.115477356709,
|
|
"request_id": 22,
|
|
"tpot_ms": 118.30690699483563,
|
|
"ttft_ms": 316.138289012585
|
|
},
|
|
{
|
|
"e2e_ms": 15387.207953555859,
|
|
"request_id": 23,
|
|
"tpot_ms": 118.31926259505173,
|
|
"ttft_ms": 360.6616039842905
|
|
},
|
|
{
|
|
"e2e_ms": 15329.406301845453,
|
|
"request_id": 24,
|
|
"tpot_ms": 118.31926259505147,
|
|
"ttft_ms": 302.8599522739164
|
|
},
|
|
{
|
|
"e2e_ms": 15375.4987780446,
|
|
"request_id": 25,
|
|
"tpot_ms": 118.33033971862962,
|
|
"ttft_ms": 347.5456337786404
|
|
},
|
|
{
|
|
"e2e_ms": 15317.697126334195,
|
|
"request_id": 26,
|
|
"tpot_ms": 118.33033971862953,
|
|
"ttft_ms": 289.74398206824503
|
|
},
|
|
{
|
|
"e2e_ms": 15362.382807838883,
|
|
"request_id": 27,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 334.400287673823
|
|
},
|
|
{
|
|
"e2e_ms": 15304.581156128477,
|
|
"request_id": 28,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 276.59863596341694
|
|
},
|
|
{
|
|
"e2e_ms": 15349.237461734056,
|
|
"request_id": 29,
|
|
"tpot_ms": 118.3192625950511,
|
|
"ttft_ms": 322.6911121625662
|
|
},
|
|
{
|
|
"e2e_ms": 15395.329937933206,
|
|
"request_id": 30,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 367.37679366725473
|
|
},
|
|
{
|
|
"e2e_ms": 15337.5282862228,
|
|
"request_id": 31,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 309.57514195684865
|
|
},
|
|
{
|
|
"e2e_ms": 15383.620762421948,
|
|
"request_id": 32,
|
|
"tpot_ms": 118.34141684220796,
|
|
"ttft_ms": 354.2608234615372
|
|
},
|
|
{
|
|
"e2e_ms": 15325.819110711542,
|
|
"request_id": 33,
|
|
"tpot_ms": 118.34141684220796,
|
|
"ttft_ms": 296.4591717511311
|
|
},
|
|
{
|
|
"e2e_ms": 15370.50479221623,
|
|
"request_id": 34,
|
|
"tpot_ms": 118.34164814850017,
|
|
"ttft_ms": 341.1154773567091
|
|
},
|
|
{
|
|
"e2e_ms": 15312.703140505824,
|
|
"request_id": 35,
|
|
"tpot_ms": 118.34164814850017,
|
|
"ttft_ms": 283.31382564630303
|
|
},
|
|
{
|
|
"e2e_ms": 15357.388822010513,
|
|
"request_id": 36,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 329.4063018454523
|
|
},
|
|
{
|
|
"e2e_ms": 15299.587170300107,
|
|
"request_id": 37,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 271.6046501350462
|
|
},
|
|
{
|
|
"e2e_ms": 15344.243475905685,
|
|
"request_id": 38,
|
|
"tpot_ms": 118.3192625950511,
|
|
"ttft_ms": 317.69712633419545
|
|
},
|
|
{
|
|
"e2e_ms": 15390.335952104835,
|
|
"request_id": 39,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 362.382807838884
|
|
},
|
|
{
|
|
"e2e_ms": 15332.53430039447,
|
|
"request_id": 40,
|
|
"tpot_ms": 118.33033971862987,
|
|
"ttft_ms": 304.5811561284779
|
|
},
|
|
{
|
|
"e2e_ms": 15378.626776593663,
|
|
"request_id": 41,
|
|
"tpot_ms": 118.34164814850084,
|
|
"ttft_ms": 349.2374617340559
|
|
},
|
|
{
|
|
"e2e_ms": 15320.8251248833,
|
|
"request_id": 42,
|
|
"tpot_ms": 118.34164814850119,
|
|
"ttft_ms": 291.4358100236498
|
|
},
|
|
{
|
|
"e2e_ms": 15365.51080638803,
|
|
"request_id": 43,
|
|
"tpot_ms": 118.33057102492307,
|
|
"ttft_ms": 337.52828622279907
|
|
},
|
|
{
|
|
"e2e_ms": 15307.709154677666,
|
|
"request_id": 44,
|
|
"tpot_ms": 118.33057102492342,
|
|
"ttft_ms": 279.726634512393
|
|
},
|
|
{
|
|
"e2e_ms": 15352.365460283294,
|
|
"request_id": 45,
|
|
"tpot_ms": 118.31926259505316,
|
|
"ttft_ms": 325.81911071154224
|
|
},
|
|
{
|
|
"e2e_ms": 15398.457936482486,
|
|
"request_id": 46,
|
|
"tpot_ms": 118.33033971863193,
|
|
"ttft_ms": 370.5047922162308
|
|
},
|
|
{
|
|
"e2e_ms": 15340.656284772123,
|
|
"request_id": 47,
|
|
"tpot_ms": 118.33033971863227,
|
|
"ttft_ms": 312.7031405058247
|
|
},
|
|
{
|
|
"e2e_ms": 15386.748760971315,
|
|
"request_id": 48,
|
|
"tpot_ms": 118.34141684221105,
|
|
"ttft_ms": 357.38882201051325
|
|
},
|
|
{
|
|
"e2e_ms": 15328.947109260951,
|
|
"request_id": 49,
|
|
"tpot_ms": 118.34141684221137,
|
|
"ttft_ms": 299.5871703001072
|
|
},
|
|
{
|
|
"e2e_ms": 15375.039585460143,
|
|
"request_id": 50,
|
|
"tpot_ms": 118.35272527208234,
|
|
"ttft_ms": 344.24347590568516
|
|
},
|
|
{
|
|
"e2e_ms": 15317.23793374978,
|
|
"request_id": 51,
|
|
"tpot_ms": 118.3527252720827,
|
|
"ttft_ms": 286.4418241952791
|
|
},
|
|
{
|
|
"e2e_ms": 15194.4223437212,
|
|
"request_id": 52,
|
|
"tpot_ms": 117.02274049863567,
|
|
"ttft_ms": 332.53430039447096
|
|
},
|
|
{
|
|
"e2e_ms": 14959.152693280232,
|
|
"request_id": 53,
|
|
"tpot_ms": 115.62535468185925,
|
|
"ttft_ms": 274.7326486841075
|
|
},
|
|
{
|
|
"e2e_ms": 14796.032219342294,
|
|
"request_id": 54,
|
|
"tpot_ms": 113.97800861778735,
|
|
"ttft_ms": 320.8251248832994
|
|
},
|
|
{
|
|
"e2e_ms": 14608.764053703837,
|
|
"request_id": 55,
|
|
"tpot_ms": 112.1516003725654,
|
|
"ttft_ms": 365.5108063880306
|
|
},
|
|
{
|
|
"e2e_ms": 14305.495223453576,
|
|
"request_id": 56,
|
|
"tpot_ms": 110.21878794311739,
|
|
"ttft_ms": 307.70915467766713
|
|
},
|
|
{
|
|
"e2e_ms": 14083.450969580696,
|
|
"request_id": 57,
|
|
"tpot_ms": 108.11878353777482,
|
|
"ttft_ms": 352.36546028329485
|
|
},
|
|
{
|
|
"e2e_ms": 13758.528543574443,
|
|
"request_id": 58,
|
|
"tpot_ms": 106.01547035434261,
|
|
"ttft_ms": 294.5638085729314
|
|
},
|
|
{
|
|
"e2e_ms": 13507.044041669204,
|
|
"request_id": 59,
|
|
"tpot_ms": 103.67234454249669,
|
|
"ttft_ms": 340.6562847721233
|
|
},
|
|
{
|
|
"e2e_ms": 13157.052579489218,
|
|
"request_id": 60,
|
|
"tpot_ms": 101.37163737344456,
|
|
"ttft_ms": 282.85463306175984
|
|
},
|
|
{
|
|
"e2e_ms": 12903.847402123616,
|
|
"request_id": 61,
|
|
"tpot_ms": 99.01496293592649,
|
|
"ttft_ms": 328.9471092609517
|
|
},
|
|
{
|
|
"e2e_ms": 12559.065843040158,
|
|
"request_id": 62,
|
|
"tpot_ms": 96.75527862590212,
|
|
"ttft_ms": 271.1454575505883
|
|
},
|
|
{
|
|
"e2e_ms": 12277.402762692504,
|
|
"request_id": 63,
|
|
"tpot_ms": 94.17452621214743,
|
|
"ttft_ms": 317.23793374978015
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "9ed2bf69d3193253248833822a357db5130e6ca29282e93e04cc92961dba468d"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 27.697141647338867,
|
|
"offered_request_rate": 1.2,
|
|
"request_metrics_sha256": "3825f14be2103486ce094e0b335c86deef683645cae8f05ef3d4beb6fb24cf95",
|
|
"request_rate_per_gpu": 0.3,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 15728.871662360909,
|
|
"request_id": 0,
|
|
"tpot_ms": 121.74246610201946,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 15904.548256233129,
|
|
"request_id": 1,
|
|
"tpot_ms": 122.72470152955445,
|
|
"ttft_ms": 318.5111619797136
|
|
},
|
|
{
|
|
"e2e_ms": 16140.50586206045,
|
|
"request_id": 2,
|
|
"tpot_ms": 124.75532551591714,
|
|
"ttft_ms": 296.5795215389744
|
|
},
|
|
{
|
|
"e2e_ms": 16211.825024161684,
|
|
"request_id": 3,
|
|
"tpot_ms": 125.48331552046938,
|
|
"ttft_ms": 275.44395306207383
|
|
},
|
|
{
|
|
"e2e_ms": 16388.84099440166,
|
|
"request_id": 4,
|
|
"tpot_ms": 126.36635826198417,
|
|
"ttft_ms": 340.31349512967114
|
|
},
|
|
{
|
|
"e2e_ms": 16460.2590055204,
|
|
"request_id": 5,
|
|
"tpot_ms": 127.08834011289514,
|
|
"ttft_ms": 320.03981118271605
|
|
},
|
|
{
|
|
"e2e_ms": 16531.726441149392,
|
|
"request_id": 6,
|
|
"tpot_ms": 127.72809453706097,
|
|
"ttft_ms": 310.25843494264916
|
|
},
|
|
{
|
|
"e2e_ms": 16603.193876778383,
|
|
"request_id": 7,
|
|
"tpot_ms": 128.3170301134443,
|
|
"ttft_ms": 306.93105237095784
|
|
},
|
|
{
|
|
"e2e_ms": 16674.731660081485,
|
|
"request_id": 8,
|
|
"tpot_ms": 128.84921275092722,
|
|
"ttft_ms": 310.8816407137276
|
|
},
|
|
{
|
|
"e2e_ms": 16746.318867894835,
|
|
"request_id": 9,
|
|
"tpot_ms": 129.28898520065795,
|
|
"ttft_ms": 326.61774741127834
|
|
},
|
|
{
|
|
"e2e_ms": 16982.978169021266,
|
|
"request_id": 10,
|
|
"tpot_ms": 130.92468009104587,
|
|
"ttft_ms": 355.5437974584379
|
|
},
|
|
{
|
|
"e2e_ms": 16948.91870099869,
|
|
"request_id": 11,
|
|
"tpot_ms": 131.099598546656,
|
|
"ttft_ms": 299.26968557337295
|
|
},
|
|
{
|
|
"e2e_ms": 17020.55604111586,
|
|
"request_id": 12,
|
|
"tpot_ms": 131.2589874619586,
|
|
"ttft_ms": 350.6646334471153
|
|
},
|
|
{
|
|
"e2e_ms": 16986.496573094286,
|
|
"request_id": 13,
|
|
"tpot_ms": 131.335760528223,
|
|
"ttft_ms": 306.854986009963
|
|
},
|
|
{
|
|
"e2e_ms": 17059.03976122884,
|
|
"request_id": 14,
|
|
"tpot_ms": 131.4398579526774,
|
|
"ttft_ms": 366.1778012388126
|
|
},
|
|
{
|
|
"e2e_ms": 17024.98029320726,
|
|
"request_id": 15,
|
|
"tpot_ms": 131.50690676411472,
|
|
"ttft_ms": 323.6031341646921
|
|
},
|
|
{
|
|
"e2e_ms": 16990.920825185683,
|
|
"request_id": 16,
|
|
"tpot_ms": 131.56847300071718,
|
|
"ttft_ms": 281.7247540946006
|
|
},
|
|
{
|
|
"e2e_ms": 17063.464013320234,
|
|
"request_id": 17,
|
|
"tpot_ms": 131.63886355765484,
|
|
"ttft_ms": 345.32834149806746
|
|
},
|
|
{
|
|
"e2e_ms": 17029.40454529866,
|
|
"request_id": 18,
|
|
"tpot_ms": 131.67812927130495,
|
|
"ttft_ms": 306.28212784293396
|
|
},
|
|
{
|
|
"e2e_ms": 16995.3450772771,
|
|
"request_id": 19,
|
|
"tpot_ms": 131.6904025418972,
|
|
"ttft_ms": 270.66395445615575
|
|
},
|
|
{
|
|
"e2e_ms": 17067.888265411675,
|
|
"request_id": 20,
|
|
"tpot_ms": 131.7034495326512,
|
|
"ttft_ms": 341.5501747649721
|
|
},
|
|
{
|
|
"e2e_ms": 17033.828797390113,
|
|
"request_id": 21,
|
|
"tpot_ms": 131.70595487135952,
|
|
"ttft_ms": 307.17252872745246
|
|
},
|
|
{
|
|
"e2e_ms": 16999.769329368555,
|
|
"request_id": 22,
|
|
"tpot_ms": 131.7084602100679,
|
|
"ttft_ms": 272.79488268993293
|
|
},
|
|
{
|
|
"e2e_ms": 17072.312517503127,
|
|
"request_id": 23,
|
|
"tpot_ms": 131.71809821033827,
|
|
"ttft_ms": 344.1140447901674
|
|
},
|
|
{
|
|
"e2e_ms": 17038.25304948157,
|
|
"request_id": 24,
|
|
"tpot_ms": 131.71982521031822,
|
|
"ttft_ms": 309.83524777115434
|
|
},
|
|
{
|
|
"e2e_ms": 17004.193581460007,
|
|
"request_id": 25,
|
|
"tpot_ms": 131.72116304093396,
|
|
"ttft_ms": 275.60587526139454
|
|
},
|
|
{
|
|
"e2e_ms": 17076.736769594583,
|
|
"request_id": 26,
|
|
"tpot_ms": 131.72963353311178,
|
|
"ttft_ms": 347.07331088938884
|
|
},
|
|
{
|
|
"e2e_ms": 17042.67730157302,
|
|
"request_id": 27,
|
|
"tpot_ms": 131.7305821943633,
|
|
"ttft_ms": 312.8933628888824
|
|
},
|
|
{
|
|
"e2e_ms": 17008.617833551463,
|
|
"request_id": 28,
|
|
"tpot_ms": 131.7309769369132,
|
|
"ttft_ms": 278.783762563485
|
|
},
|
|
{
|
|
"e2e_ms": 17081.161021686035,
|
|
"request_id": 29,
|
|
"tpot_ms": 131.73850434102513,
|
|
"ttft_ms": 350.37097037584175
|
|
},
|
|
{
|
|
"e2e_ms": 17047.986478516996,
|
|
"request_id": 30,
|
|
"tpot_ms": 131.74547225324986,
|
|
"ttft_ms": 316.3115023542638
|
|
},
|
|
{
|
|
"e2e_ms": 17013.927010495438,
|
|
"request_id": 31,
|
|
"tpot_ms": 131.74547225325,
|
|
"ttft_ms": 282.25203433268575
|
|
},
|
|
{
|
|
"e2e_ms": 17086.47019863001,
|
|
"request_id": 32,
|
|
"tpot_ms": 131.75260491481217,
|
|
"ttft_ms": 353.8893744488618
|
|
},
|
|
{
|
|
"e2e_ms": 17053.29565546097,
|
|
"request_id": 33,
|
|
"tpot_ms": 131.7595728270369,
|
|
"ttft_ms": 319.82990642728385
|
|
},
|
|
{
|
|
"e2e_ms": 17019.236187439412,
|
|
"request_id": 34,
|
|
"tpot_ms": 131.75957282703703,
|
|
"ttft_ms": 285.7704384057058
|
|
},
|
|
{
|
|
"e2e_ms": 17091.779375573984,
|
|
"request_id": 35,
|
|
"tpot_ms": 131.7595728270372,
|
|
"ttft_ms": 358.31362654026225
|
|
},
|
|
{
|
|
"e2e_ms": 17058.604832404944,
|
|
"request_id": 36,
|
|
"tpot_ms": 131.7665407392619,
|
|
"ttft_ms": 324.2541585186842
|
|
},
|
|
{
|
|
"e2e_ms": 17024.545364383386,
|
|
"request_id": 37,
|
|
"tpot_ms": 131.76654073926207,
|
|
"ttft_ms": 290.1946904971062
|
|
},
|
|
{
|
|
"e2e_ms": 17097.08855251796,
|
|
"request_id": 38,
|
|
"tpot_ms": 131.76654073926218,
|
|
"ttft_ms": 362.7378786316626
|
|
},
|
|
{
|
|
"e2e_ms": 17063.934932514778,
|
|
"request_id": 39,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 328.6784106101024
|
|
},
|
|
{
|
|
"e2e_ms": 17029.875464493216,
|
|
"request_id": 40,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 294.61894258854215
|
|
},
|
|
{
|
|
"e2e_ms": 17102.41865262779,
|
|
"request_id": 41,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 367.1621307231163
|
|
},
|
|
{
|
|
"e2e_ms": 17069.265032624608,
|
|
"request_id": 42,
|
|
"tpot_ms": 131.78080606238623,
|
|
"ttft_ms": 333.1026627015561
|
|
},
|
|
{
|
|
"e2e_ms": 17035.205564603046,
|
|
"request_id": 43,
|
|
"tpot_ms": 131.78080606238623,
|
|
"ttft_ms": 299.04319467999585
|
|
},
|
|
{
|
|
"e2e_ms": 16941.812657737883,
|
|
"request_id": 44,
|
|
"tpot_ms": 130.47422263719145,
|
|
"ttft_ms": 371.58638281457
|
|
},
|
|
{
|
|
"e2e_ms": 16742.48916517699,
|
|
"request_id": 45,
|
|
"tpot_ms": 129.1729311053857,
|
|
"ttft_ms": 337.5269147930098
|
|
},
|
|
{
|
|
"e2e_ms": 16538.03643831958,
|
|
"request_id": 46,
|
|
"tpot_ms": 127.83125190195379,
|
|
"ttft_ms": 303.46744677144954
|
|
},
|
|
{
|
|
"e2e_ms": 16435.957848641778,
|
|
"request_id": 47,
|
|
"tpot_ms": 126.45627727350987,
|
|
"ttft_ms": 376.0106349060237
|
|
},
|
|
{
|
|
"e2e_ms": 16228.476901947985,
|
|
"request_id": 48,
|
|
"tpot_ms": 125.09075381939779,
|
|
"ttft_ms": 341.9511668844635
|
|
},
|
|
{
|
|
"e2e_ms": 16020.052127617348,
|
|
"request_id": 49,
|
|
"tpot_ms": 123.7177986516098,
|
|
"ttft_ms": 307.89169886290324
|
|
},
|
|
{
|
|
"e2e_ms": 15809.427558622445,
|
|
"request_id": 50,
|
|
"tpot_ms": 122.32055435377619,
|
|
"ttft_ms": 274.71715569286914
|
|
},
|
|
{
|
|
"e2e_ms": 15703.511289947706,
|
|
"request_id": 51,
|
|
"tpot_ms": 120.91536178046667,
|
|
"ttft_ms": 347.26034382843807
|
|
},
|
|
{
|
|
"e2e_ms": 15485.101955905271,
|
|
"request_id": 52,
|
|
"tpot_ms": 119.46378803227081,
|
|
"ttft_ms": 313.20087580687783
|
|
},
|
|
{
|
|
"e2e_ms": 15260.624457846197,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.95746555282956,
|
|
"ttft_ms": 280.02633263684373
|
|
},
|
|
{
|
|
"e2e_ms": 15118.941732007286,
|
|
"request_id": 54,
|
|
"tpot_ms": 116.27064733255806,
|
|
"ttft_ms": 352.56952077241266
|
|
},
|
|
{
|
|
"e2e_ms": 14865.280308326035,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.54149807539514,
|
|
"ttft_ms": 318.5100527508524
|
|
},
|
|
{
|
|
"e2e_ms": 14599.985676841925,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.71378084457564,
|
|
"ttft_ms": 285.3355095808183
|
|
},
|
|
{
|
|
"e2e_ms": 14420.806000135271,
|
|
"request_id": 57,
|
|
"tpot_ms": 110.73171104266838,
|
|
"ttft_ms": 357.87869771638725
|
|
},
|
|
{
|
|
"e2e_ms": 14142.519211198518,
|
|
"request_id": 58,
|
|
"tpot_ms": 108.80866127168261,
|
|
"ttft_ms": 323.819229694827
|
|
},
|
|
{
|
|
"e2e_ms": 13853.830695007076,
|
|
"request_id": 59,
|
|
"tpot_ms": 106.79657547493251,
|
|
"ttft_ms": 290.6656096906488
|
|
},
|
|
{
|
|
"e2e_ms": 13650.503338148035,
|
|
"request_id": 60,
|
|
"tpot_ms": 104.62436645922692,
|
|
"ttft_ms": 363.20879782621773
|
|
},
|
|
{
|
|
"e2e_ms": 13360.876271706531,
|
|
"request_id": 61,
|
|
"tpot_ms": 102.61202316458169,
|
|
"ttft_ms": 329.1493298046575
|
|
},
|
|
{
|
|
"e2e_ms": 13070.674002922458,
|
|
"request_id": 62,
|
|
"tpot_ms": 100.58801805607857,
|
|
"ttft_ms": 295.9957098004793
|
|
},
|
|
{
|
|
"e2e_ms": 12853.555073004585,
|
|
"request_id": 63,
|
|
"tpot_ms": 98.30721397691761,
|
|
"ttft_ms": 368.5388979360482
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.1875,
|
|
"passed": 12
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "3e07b28ac20428d11b8d83064d68561f03804f8adf37111de43398fdd8d94138"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 28.44595432281494,
|
|
"offered_request_rate": 1.6,
|
|
"request_metrics_sha256": "b7d73a1cbef8ca659f753135a4944cb3023d3a2424a19a741ae2198f39c2d172",
|
|
"request_rate_per_gpu": 0.4,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 17513.570713343794,
|
|
"request_id": 0,
|
|
"tpot_ms": 135.79521453495556,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 17840.282304762335,
|
|
"request_id": 1,
|
|
"tpot_ms": 137.71064841138914,
|
|
"ttft_ms": 351.0299565159147
|
|
},
|
|
{
|
|
"e2e_ms": 17895.63618825912,
|
|
"request_id": 2,
|
|
"tpot_ms": 138.10245652748367,
|
|
"ttft_ms": 356.6242092686922
|
|
},
|
|
{
|
|
"e2e_ms": 17848.77057711112,
|
|
"request_id": 3,
|
|
"tpot_ms": 138.39719968518088,
|
|
"ttft_ms": 272.32621709314975
|
|
},
|
|
{
|
|
"e2e_ms": 17904.1244606079,
|
|
"request_id": 4,
|
|
"tpot_ms": 138.80479077491768,
|
|
"ttft_ms": 275.9160321933569
|
|
},
|
|
{
|
|
"e2e_ms": 17959.478344104686,
|
|
"request_id": 5,
|
|
"tpot_ms": 139.18735062645055,
|
|
"ttft_ms": 282.6848145454632
|
|
},
|
|
{
|
|
"e2e_ms": 18014.921946878916,
|
|
"request_id": 6,
|
|
"tpot_ms": 139.5152186600215,
|
|
"ttft_ms": 296.48917705618504
|
|
},
|
|
{
|
|
"e2e_ms": 18070.36554965315,
|
|
"request_id": 7,
|
|
"tpot_ms": 139.8086106441042,
|
|
"ttft_ms": 314.671997851919
|
|
},
|
|
{
|
|
"e2e_ms": 18125.898871704834,
|
|
"request_id": 8,
|
|
"tpot_ms": 140.0646556798527,
|
|
"ttft_ms": 337.68760036353916
|
|
},
|
|
{
|
|
"e2e_ms": 18079.30241838919,
|
|
"request_id": 9,
|
|
"tpot_ms": 140.21470125635153,
|
|
"ttft_ms": 272.0353588325457
|
|
},
|
|
{
|
|
"e2e_ms": 18134.83860521682,
|
|
"request_id": 10,
|
|
"tpot_ms": 140.33922237580643,
|
|
"ttft_ms": 311.75736348940484
|
|
},
|
|
{
|
|
"e2e_ms": 18190.4645113219,
|
|
"request_id": 11,
|
|
"tpot_ms": 140.37869703173442,
|
|
"ttft_ms": 362.36998829163
|
|
},
|
|
{
|
|
"e2e_ms": 18143.870922782207,
|
|
"request_id": 12,
|
|
"tpot_ms": 140.38698051363622,
|
|
"ttft_ms": 314.7243975504068
|
|
},
|
|
{
|
|
"e2e_ms": 18097.274469466563,
|
|
"request_id": 13,
|
|
"tpot_ms": 140.36074521133403,
|
|
"ttft_ms": 271.4598276271438
|
|
},
|
|
{
|
|
"e2e_ms": 18152.810656294194,
|
|
"request_id": 14,
|
|
"tpot_ms": 140.32455966134754,
|
|
"ttft_ms": 331.5915793030584
|
|
},
|
|
{
|
|
"e2e_ms": 18106.217067754496,
|
|
"request_id": 15,
|
|
"tpot_ms": 140.29231877139563,
|
|
"ttft_ms": 289.0925837872533
|
|
},
|
|
{
|
|
"e2e_ms": 18161.753254582127,
|
|
"request_id": 16,
|
|
"tpot_ms": 140.24009333090902,
|
|
"ttft_ms": 351.2614015566804
|
|
},
|
|
{
|
|
"e2e_ms": 18115.15966604243,
|
|
"request_id": 17,
|
|
"tpot_ms": 140.19912163029534,
|
|
"ttft_ms": 309.87121899492286
|
|
},
|
|
{
|
|
"e2e_ms": 18170.69585287006,
|
|
"request_id": 18,
|
|
"tpot_ms": 140.12856604535176,
|
|
"ttft_ms": 374.3679651103839
|
|
},
|
|
{
|
|
"e2e_ms": 18124.102264330366,
|
|
"request_id": 19,
|
|
"tpot_ms": 140.05569589908265,
|
|
"ttft_ms": 337.0288851468697
|
|
},
|
|
{
|
|
"e2e_ms": 18077.42933293013,
|
|
"request_id": 20,
|
|
"tpot_ms": 139.98073600581233,
|
|
"ttft_ms": 299.87586019196044
|
|
},
|
|
{
|
|
"e2e_ms": 18132.96551975776,
|
|
"request_id": 21,
|
|
"tpot_ms": 139.8495015438229,
|
|
"ttft_ms": 372.0788236922541
|
|
},
|
|
{
|
|
"e2e_ms": 18086.369066442116,
|
|
"request_id": 22,
|
|
"tpot_ms": 139.76199603449132,
|
|
"ttft_ms": 336.5955700617178
|
|
},
|
|
{
|
|
"e2e_ms": 18039.682893849018,
|
|
"request_id": 23,
|
|
"tpot_ms": 139.67091992877667,
|
|
"ttft_ms": 301.4760628943822
|
|
},
|
|
{
|
|
"e2e_ms": 18095.21908067664,
|
|
"request_id": 24,
|
|
"tpot_ms": 139.54200325642663,
|
|
"ttft_ms": 373.38466711045726
|
|
},
|
|
{
|
|
"e2e_ms": 18048.532908083544,
|
|
"request_id": 25,
|
|
"tpot_ms": 139.5517742330953,
|
|
"ttft_ms": 325.4575804804407
|
|
},
|
|
{
|
|
"e2e_ms": 18001.846735490446,
|
|
"request_id": 26,
|
|
"tpot_ms": 139.5610841122856,
|
|
"ttft_ms": 277.589053230173
|
|
},
|
|
{
|
|
"e2e_ms": 18057.290338264673,
|
|
"request_id": 27,
|
|
"tpot_ms": 139.56778701592805,
|
|
"ttft_ms": 332.18138724181046
|
|
},
|
|
{
|
|
"e2e_ms": 18010.60416567157,
|
|
"request_id": 28,
|
|
"tpot_ms": 139.57154350494935,
|
|
"ttft_ms": 285.01814054300564
|
|
},
|
|
{
|
|
"e2e_ms": 18066.06100963866,
|
|
"request_id": 29,
|
|
"tpot_ms": 139.5730606683175,
|
|
"ttft_ms": 340.2823047623365
|
|
},
|
|
{
|
|
"e2e_ms": 18019.37483704556,
|
|
"request_id": 30,
|
|
"tpot_ms": 139.57447357032456,
|
|
"ttft_ms": 293.41669361433986
|
|
},
|
|
{
|
|
"e2e_ms": 18074.831681012645,
|
|
"request_id": 31,
|
|
"tpot_ms": 139.57528428268915,
|
|
"ttft_ms": 348.7705771111216
|
|
},
|
|
{
|
|
"e2e_ms": 18028.145508419548,
|
|
"request_id": 32,
|
|
"tpot_ms": 139.57669718469626,
|
|
"ttft_ms": 301.904965963125
|
|
},
|
|
{
|
|
"e2e_ms": 18083.678830471228,
|
|
"request_id": 33,
|
|
"tpot_ms": 139.5781100867033,
|
|
"ttft_ms": 357.25884945990674
|
|
},
|
|
{
|
|
"e2e_ms": 18036.99265787813,
|
|
"request_id": 34,
|
|
"tpot_ms": 139.57881653770684,
|
|
"ttft_ms": 310.48295758936104
|
|
},
|
|
{
|
|
"e2e_ms": 18092.528844705754,
|
|
"request_id": 35,
|
|
"tpot_ms": 139.57954554600127,
|
|
"ttft_ms": 365.9265603635937
|
|
},
|
|
{
|
|
"e2e_ms": 17876.05367572695,
|
|
"request_id": 36,
|
|
"tpot_ms": 138.24262431461773,
|
|
"ttft_ms": 319.240387770499
|
|
},
|
|
{
|
|
"e2e_ms": 17659.21004164318,
|
|
"request_id": 37,
|
|
"tpot_ms": 136.90209533219152,
|
|
"ttft_ms": 272.64393445485524
|
|
},
|
|
{
|
|
"e2e_ms": 17543.75228976072,
|
|
"request_id": 38,
|
|
"tpot_ms": 135.55570892326125,
|
|
"ttft_ms": 328.1772565065388
|
|
},
|
|
{
|
|
"e2e_ms": 17326.367174433202,
|
|
"request_id": 39,
|
|
"tpot_ms": 134.21089375170362,
|
|
"ttft_ms": 281.5836679668422
|
|
},
|
|
{
|
|
"e2e_ms": 17233.832476815856,
|
|
"request_id": 40,
|
|
"tpot_ms": 133.04427482475538,
|
|
"ttft_ms": 337.20957407192384
|
|
},
|
|
{
|
|
"e2e_ms": 17034.265611203216,
|
|
"request_id": 41,
|
|
"tpot_ms": 131.83976083205505,
|
|
"ttft_ms": 290.6159855322272
|
|
},
|
|
{
|
|
"e2e_ms": 16940.502121615402,
|
|
"request_id": 42,
|
|
"tpot_ms": 130.66419538607474,
|
|
"ttft_ms": 346.14930758391085
|
|
},
|
|
{
|
|
"e2e_ms": 16741.97064984836,
|
|
"request_id": 43,
|
|
"tpot_ms": 129.46783410082003,
|
|
"ttft_ms": 299.5557190442142
|
|
},
|
|
{
|
|
"e2e_ms": 16645.559306371404,
|
|
"request_id": 44,
|
|
"tpot_ms": 128.27139685432724,
|
|
"ttft_ms": 355.0919058718449
|
|
},
|
|
{
|
|
"e2e_ms": 16443.232533552808,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.04515130882409,
|
|
"ttft_ms": 308.4983173321483
|
|
},
|
|
{
|
|
"e2e_ms": 16342.299705615802,
|
|
"request_id": 46,
|
|
"tpot_ms": 125.81311182248837,
|
|
"ttft_ms": 364.034504159779
|
|
},
|
|
{
|
|
"e2e_ms": 16135.159559133626,
|
|
"request_id": 47,
|
|
"tpot_ms": 124.54896569695704,
|
|
"ttft_ms": 317.44091562008236
|
|
},
|
|
{
|
|
"e2e_ms": 16031.7032146663,
|
|
"request_id": 48,
|
|
"tpot_ms": 123.29695174036006,
|
|
"ttft_ms": 372.9903436405735
|
|
},
|
|
{
|
|
"e2e_ms": 15823.308920667878,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.0236594458299,
|
|
"ttft_ms": 326.3041710474788
|
|
},
|
|
{
|
|
"e2e_ms": 15613.44809689394,
|
|
"request_id": 50,
|
|
"tpot_ms": 120.73809066445794,
|
|
"ttft_ms": 279.71058250778214
|
|
},
|
|
{
|
|
"e2e_ms": 15508.031714506984,
|
|
"request_id": 51,
|
|
"tpot_ms": 119.47077015706711,
|
|
"ttft_ms": 335.2439045594622
|
|
},
|
|
{
|
|
"e2e_ms": 15292.981047367362,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.14506547559841,
|
|
"ttft_ms": 288.5577319663639
|
|
},
|
|
{
|
|
"e2e_ms": 15175.360910096466,
|
|
"request_id": 53,
|
|
"tpot_ms": 116.78162985277542,
|
|
"ttft_ms": 344.09391879398754
|
|
},
|
|
{
|
|
"e2e_ms": 14945.589828664517,
|
|
"request_id": 54,
|
|
"tpot_ms": 115.34001639735139,
|
|
"ttft_ms": 297.4077462008893
|
|
},
|
|
{
|
|
"e2e_ms": 14803.64975337431,
|
|
"request_id": 55,
|
|
"tpot_ms": 113.78581420786767,
|
|
"ttft_ms": 352.85134897511483
|
|
},
|
|
{
|
|
"e2e_ms": 14557.342221274233,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.21399247946627,
|
|
"ttft_ms": 306.16517638201657
|
|
},
|
|
{
|
|
"e2e_ms": 14399.594833150197,
|
|
"request_id": 57,
|
|
"tpot_ms": 110.5352189984338,
|
|
"ttft_ms": 361.6220203491025
|
|
},
|
|
{
|
|
"e2e_ms": 14144.625862747695,
|
|
"request_id": 58,
|
|
"tpot_ms": 108.89519696843851,
|
|
"ttft_ms": 314.93584775600425
|
|
},
|
|
{
|
|
"e2e_ms": 13973.496543699825,
|
|
"request_id": 59,
|
|
"tpot_ms": 107.1110539525727,
|
|
"ttft_ms": 370.3926917230902
|
|
},
|
|
{
|
|
"e2e_ms": 13708.4999588278,
|
|
"request_id": 60,
|
|
"tpot_ms": 105.3920743283292,
|
|
"ttft_ms": 323.70651912999193
|
|
},
|
|
{
|
|
"e2e_ms": 13445.970803422128,
|
|
"request_id": 61,
|
|
"tpot_ms": 103.6925232825609,
|
|
"ttft_ms": 277.02034653689367
|
|
},
|
|
{
|
|
"e2e_ms": 13273.580023880399,
|
|
"request_id": 62,
|
|
"tpot_ms": 101.89784531725846,
|
|
"ttft_ms": 332.5536685885737
|
|
},
|
|
{
|
|
"e2e_ms": 13000.702540117643,
|
|
"request_id": 63,
|
|
"tpot_ms": 100.1168113710407,
|
|
"ttft_ms": 285.86749599547545
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.203125,
|
|
"passed": 13
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "850087527f8ec5b333ae4bb00c5837c85f7dafca77de3d8c80820e071da8dfbb"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 33.35526180267334,
|
|
"offered_request_rate": 2.4,
|
|
"request_metrics_sha256": "3b42135fbabbd54647284962ccc9fdf0b900fbc0b6ed0bdecf03477d5c409d18",
|
|
"request_rate_per_gpu": 0.6,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 21942.173036302996,
|
|
"request_id": 0,
|
|
"tpot_ms": 170.66609896770518,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 22008.341136862466,
|
|
"request_id": 1,
|
|
"tpot_ms": 170.96613901739173,
|
|
"ttft_ms": 295.64148165371654
|
|
},
|
|
{
|
|
"e2e_ms": 22074.54102422534,
|
|
"request_id": 2,
|
|
"tpot_ms": 171.22660942224667,
|
|
"ttft_ms": 328.7616276000109
|
|
},
|
|
{
|
|
"e2e_ms": 22036.227907899276,
|
|
"request_id": 3,
|
|
"tpot_ms": 171.37344968894675,
|
|
"ttft_ms": 271.7997974030406
|
|
},
|
|
{
|
|
"e2e_ms": 22102.42779526115,
|
|
"request_id": 4,
|
|
"tpot_ms": 171.64131409153669,
|
|
"ttft_ms": 303.98090563598765
|
|
},
|
|
{
|
|
"e2e_ms": 22168.65946942643,
|
|
"request_id": 5,
|
|
"tpot_ms": 171.89659303259273,
|
|
"ttft_ms": 337.79215428715537
|
|
},
|
|
{
|
|
"e2e_ms": 22130.37813990278,
|
|
"request_id": 6,
|
|
"tpot_ms": 172.0281077294899,
|
|
"ttft_ms": 282.80845825756097
|
|
},
|
|
{
|
|
"e2e_ms": 22365.99994575454,
|
|
"request_id": 7,
|
|
"tpot_ms": 173.57085404199856,
|
|
"ttft_ms": 322.5014824207202
|
|
},
|
|
{
|
|
"e2e_ms": 22158.3592864644,
|
|
"request_id": 8,
|
|
"tpot_ms": 172.3496956452424,
|
|
"ttft_ms": 269.9479395186155
|
|
},
|
|
{
|
|
"e2e_ms": 22393.981092316164,
|
|
"request_id": 9,
|
|
"tpot_ms": 173.8419366523574,
|
|
"ttft_ms": 316.0551374667744
|
|
},
|
|
{
|
|
"e2e_ms": 22460.243568400445,
|
|
"request_id": 10,
|
|
"tpot_ms": 173.96590922572528,
|
|
"ttft_ms": 366.57309673333316
|
|
},
|
|
{
|
|
"e2e_ms": 22252.602909110316,
|
|
"request_id": 11,
|
|
"tpot_ms": 172.6883751510306,
|
|
"ttft_ms": 321.1792649294285
|
|
},
|
|
{
|
|
"e2e_ms": 22044.894176229613,
|
|
"request_id": 12,
|
|
"tpot_ms": 171.40112664047544,
|
|
"ttft_ms": 276.9510928892318
|
|
},
|
|
{
|
|
"e2e_ms": 21941.653543442477,
|
|
"request_id": 13,
|
|
"tpot_ms": 170.1065182287643,
|
|
"ttft_ms": 338.1257283894126
|
|
},
|
|
{
|
|
"e2e_ms": 21734.138079618475,
|
|
"request_id": 14,
|
|
"tpot_ms": 168.80971544527205,
|
|
"ttft_ms": 295.30421806892537
|
|
},
|
|
{
|
|
"e2e_ms": 21631.119179231846,
|
|
"request_id": 15,
|
|
"tpot_ms": 167.51267286797687,
|
|
"ttft_ms": 357.0097249987798
|
|
},
|
|
{
|
|
"e2e_ms": 21420.354658102242,
|
|
"request_id": 16,
|
|
"tpot_ms": 166.18662353135022,
|
|
"ttft_ms": 314.65346962076256
|
|
},
|
|
{
|
|
"e2e_ms": 21209.648295563282,
|
|
"request_id": 17,
|
|
"tpot_ms": 164.85776701341018,
|
|
"ttft_ms": 272.7118848601879
|
|
},
|
|
{
|
|
"e2e_ms": 21101.69867612793,
|
|
"request_id": 18,
|
|
"tpot_ms": 163.50974377121332,
|
|
"ttft_ms": 335.9612171838391
|
|
},
|
|
{
|
|
"e2e_ms": 20890.778369055995,
|
|
"request_id": 19,
|
|
"tpot_ms": 162.16628083276893,
|
|
"ttft_ms": 295.66070329433813
|
|
},
|
|
{
|
|
"e2e_ms": 20782.558056179907,
|
|
"request_id": 20,
|
|
"tpot_ms": 160.7985342727971,
|
|
"ttft_ms": 361.14420353467835
|
|
},
|
|
{
|
|
"e2e_ms": 20571.512469390585,
|
|
"request_id": 21,
|
|
"tpot_ms": 159.44598375979533,
|
|
"ttft_ms": 321.8725318965792
|
|
},
|
|
{
|
|
"e2e_ms": 20360.405278761264,
|
|
"request_id": 22,
|
|
"tpot_ms": 158.09565605925098,
|
|
"ttft_ms": 282.2569592363866
|
|
},
|
|
{
|
|
"e2e_ms": 20251.586130359967,
|
|
"request_id": 23,
|
|
"tpot_ms": 156.71227679254048,
|
|
"ttft_ms": 349.12697770732314
|
|
},
|
|
{
|
|
"e2e_ms": 20042.10448060375,
|
|
"request_id": 24,
|
|
"tpot_ms": 155.37142770924103,
|
|
"ttft_ms": 309.93316153013774
|
|
},
|
|
{
|
|
"e2e_ms": 19934.008109113514,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.05960281316982,
|
|
"ttft_ms": 368.43855184094565
|
|
},
|
|
{
|
|
"e2e_ms": 19722.984661067087,
|
|
"request_id": 26,
|
|
"tpot_ms": 152.73931080974432,
|
|
"ttft_ms": 325.0921882295579
|
|
},
|
|
{
|
|
"e2e_ms": 19511.768043076878,
|
|
"request_id": 27,
|
|
"tpot_ms": 151.41621155599998,
|
|
"ttft_ms": 281.9091754648788
|
|
},
|
|
{
|
|
"e2e_ms": 19403.203575383635,
|
|
"request_id": 28,
|
|
"tpot_ms": 150.09414530000853,
|
|
"ttft_ms": 341.2471222825513
|
|
},
|
|
{
|
|
"e2e_ms": 19191.714653347688,
|
|
"request_id": 29,
|
|
"tpot_ms": 148.76487274343572,
|
|
"ttft_ms": 298.57581493135046
|
|
},
|
|
{
|
|
"e2e_ms": 19082.708845359608,
|
|
"request_id": 30,
|
|
"tpot_ms": 147.43434256455518,
|
|
"ttft_ms": 358.5473396611025
|
|
},
|
|
{
|
|
"e2e_ms": 18871.078709760895,
|
|
"request_id": 31,
|
|
"tpot_ms": 146.10277278695904,
|
|
"ttft_ms": 316.0265658170971
|
|
},
|
|
{
|
|
"e2e_ms": 18659.370447496967,
|
|
"request_id": 32,
|
|
"tpot_ms": 144.7695321688903,
|
|
"ttft_ms": 273.6398620479008
|
|
},
|
|
{
|
|
"e2e_ms": 18549.637648096734,
|
|
"request_id": 33,
|
|
"tpot_ms": 143.43132538741386,
|
|
"ttft_ms": 333.85932389517234
|
|
},
|
|
{
|
|
"e2e_ms": 18337.808235788358,
|
|
"request_id": 34,
|
|
"tpot_ms": 142.0958670025312,
|
|
"ttft_ms": 291.63312646689656
|
|
},
|
|
{
|
|
"e2e_ms": 18227.80864738452,
|
|
"request_id": 35,
|
|
"tpot_ms": 140.75352098043697,
|
|
"ttft_ms": 352.1114828690255
|
|
},
|
|
{
|
|
"e2e_ms": 18015.40726100407,
|
|
"request_id": 36,
|
|
"tpot_ms": 139.41149279948888,
|
|
"ttft_ms": 310.14767546898446
|
|
},
|
|
{
|
|
"e2e_ms": 17904.857868774245,
|
|
"request_id": 37,
|
|
"tpot_ms": 138.06211837292886,
|
|
"ttft_ms": 370.9688354122758
|
|
},
|
|
{
|
|
"e2e_ms": 17692.008101355255,
|
|
"request_id": 38,
|
|
"tpot_ms": 136.71510258553306,
|
|
"ttft_ms": 329.1900729925583
|
|
},
|
|
{
|
|
"e2e_ms": 17479.148877024498,
|
|
"request_id": 39,
|
|
"tpot_ms": 135.36749298254742,
|
|
"ttft_ms": 287.47726824097344
|
|
},
|
|
{
|
|
"e2e_ms": 17381.961391787096,
|
|
"request_id": 40,
|
|
"tpot_ms": 134.1089349239123,
|
|
"ttft_ms": 350.1266564502323
|
|
},
|
|
{
|
|
"e2e_ms": 17178.01129231477,
|
|
"request_id": 41,
|
|
"tpot_ms": 132.8327622110564,
|
|
"ttft_ms": 308.2504915106092
|
|
},
|
|
{
|
|
"e2e_ms": 17080.086531895085,
|
|
"request_id": 42,
|
|
"tpot_ms": 131.5805170059729,
|
|
"ttft_ms": 369.36087213652866
|
|
},
|
|
{
|
|
"e2e_ms": 16876.654129344566,
|
|
"request_id": 43,
|
|
"tpot_ms": 130.30640250229436,
|
|
"ttft_ms": 327.74101155318294
|
|
},
|
|
{
|
|
"e2e_ms": 16671.422925287778,
|
|
"request_id": 44,
|
|
"tpot_ms": 129.0178430005707,
|
|
"ttft_ms": 286.15686421529676
|
|
},
|
|
{
|
|
"e2e_ms": 16571.761179006833,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.74972161297816,
|
|
"ttft_ms": 347.5465341586066
|
|
},
|
|
{
|
|
"e2e_ms": 16364.721381165033,
|
|
"request_id": 46,
|
|
"tpot_ms": 126.44572566423672,
|
|
"ttft_ms": 306.1142218069683
|
|
},
|
|
{
|
|
"e2e_ms": 16261.449604637399,
|
|
"request_id": 47,
|
|
"tpot_ms": 125.14700916940275,
|
|
"ttft_ms": 367.77944012325037
|
|
},
|
|
{
|
|
"e2e_ms": 16053.400400183471,
|
|
"request_id": 48,
|
|
"tpot_ms": 123.8340136000885,
|
|
"ttft_ms": 326.48067297223093
|
|
},
|
|
{
|
|
"e2e_ms": 15845.03658651726,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.50141376236263,
|
|
"ttft_ms": 287.35703869720555
|
|
},
|
|
{
|
|
"e2e_ms": 15739.724302020804,
|
|
"request_id": 50,
|
|
"tpot_ms": 121.15600683324593,
|
|
"ttft_ms": 352.9114341985711
|
|
},
|
|
{
|
|
"e2e_ms": 15530.891082399023,
|
|
"request_id": 51,
|
|
"tpot_ms": 119.81946138926746,
|
|
"ttft_ms": 313.8194859620569
|
|
},
|
|
{
|
|
"e2e_ms": 15319.199082162215,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.45427332697808,
|
|
"ttft_ms": 275.5063696359983
|
|
},
|
|
{
|
|
"e2e_ms": 15206.960333133673,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.04949498375757,
|
|
"ttft_ms": 341.6744701964625
|
|
},
|
|
{
|
|
"e2e_ms": 14987.9081257507,
|
|
"request_id": 54,
|
|
"tpot_ms": 115.62635253449054,
|
|
"ttft_ms": 303.36135387040383
|
|
},
|
|
{
|
|
"e2e_ms": 14861.077413909577,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.10642655651417,
|
|
"ttft_ms": 369.5612412322795
|
|
},
|
|
{
|
|
"e2e_ms": 14633.756981193535,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.61817997075843,
|
|
"ttft_ms": 331.2481249072157
|
|
},
|
|
{
|
|
"e2e_ms": 14403.991359276923,
|
|
"request_id": 57,
|
|
"tpot_ms": 111.11042963695559,
|
|
"ttft_ms": 292.9667953835633
|
|
},
|
|
{
|
|
"e2e_ms": 14264.847964808048,
|
|
"request_id": 58,
|
|
"tpot_ms": 109.49330311228506,
|
|
"ttft_ms": 359.1984695478452
|
|
},
|
|
{
|
|
"e2e_ms": 14029.729570522895,
|
|
"request_id": 59,
|
|
"tpot_ms": 107.94340496454889,
|
|
"ttft_ms": 320.91714002518756
|
|
},
|
|
{
|
|
"e2e_ms": 13793.064611419886,
|
|
"request_id": 60,
|
|
"tpot_ms": 106.38108660628622,
|
|
"ttft_ms": 282.66661242153646
|
|
},
|
|
{
|
|
"e2e_ms": 13648.251078198633,
|
|
"request_id": 61,
|
|
"tpot_ms": 104.71931331978594,
|
|
"ttft_ms": 348.89828658581837
|
|
},
|
|
{
|
|
"e2e_ms": 13412.62809971594,
|
|
"request_id": 62,
|
|
"tpot_ms": 103.16519953332896,
|
|
"ttft_ms": 310.647758983162
|
|
},
|
|
{
|
|
"e2e_ms": 13260.053320226874,
|
|
"request_id": 63,
|
|
"tpot_ms": 101.44207153668842,
|
|
"ttft_ms": 376.9102350674451
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.203125,
|
|
"passed": 13
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.546875,
|
|
"passed": 35
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "0b9f30d883de83b4af69dd114bd5d2bb974b7fd1f01d45f913bf5049e4b72a86"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 34.759880781173706,
|
|
"offered_request_rate": 3.2,
|
|
"request_metrics_sha256": "bfba1ae71cfb6d0595614b14bc49ac47591bfad6792be0a3d98d2c98b16012e8",
|
|
"request_rate_per_gpu": 0.8,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 24003.81066522886,
|
|
"request_id": 0,
|
|
"tpot_ms": 186.89946612460176,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 23901.058156177503,
|
|
"request_id": 1,
|
|
"tpot_ms": 185.74139588389912,
|
|
"ttft_ms": 311.9008789223171
|
|
},
|
|
{
|
|
"e2e_ms": 23798.190109844723,
|
|
"request_id": 2,
|
|
"tpot_ms": 184.56225305811301,
|
|
"ttft_ms": 358.7839714643697
|
|
},
|
|
{
|
|
"e2e_ms": 23590.44639829378,
|
|
"request_id": 3,
|
|
"tpot_ms": 183.2673714648551,
|
|
"ttft_ms": 315.49022225718517
|
|
},
|
|
{
|
|
"e2e_ms": 23382.70131446869,
|
|
"request_id": 4,
|
|
"tpot_ms": 181.9732705315891,
|
|
"ttft_ms": 272.09595695687216
|
|
},
|
|
{
|
|
"e2e_ms": 23279.64954297851,
|
|
"request_id": 5,
|
|
"tpot_ms": 180.78914255292864,
|
|
"ttft_ms": 319.4284387565705
|
|
},
|
|
{
|
|
"e2e_ms": 23071.973865900134,
|
|
"request_id": 6,
|
|
"tpot_ms": 179.49398625037213,
|
|
"ttft_ms": 276.2376121028707
|
|
},
|
|
{
|
|
"e2e_ms": 22969.022726489202,
|
|
"request_id": 7,
|
|
"tpot_ms": 178.2861180199918,
|
|
"ttft_ms": 326.6857379502426
|
|
},
|
|
{
|
|
"e2e_ms": 22761.29942646063,
|
|
"request_id": 8,
|
|
"tpot_ms": 176.98924979786668,
|
|
"ttft_ms": 283.66470213156344
|
|
},
|
|
{
|
|
"e2e_ms": 22658.264587165988,
|
|
"request_id": 9,
|
|
"tpot_ms": 175.7551174352214,
|
|
"ttft_ms": 337.36467289287384
|
|
},
|
|
{
|
|
"e2e_ms": 22450.304224673095,
|
|
"request_id": 10,
|
|
"tpot_ms": 174.4550575162989,
|
|
"ttft_ms": 294.51192010313275
|
|
},
|
|
{
|
|
"e2e_ms": 22346.821408551063,
|
|
"request_id": 11,
|
|
"tpot_ms": 173.1774539652428,
|
|
"ttft_ms": 353.28475496522805
|
|
},
|
|
{
|
|
"e2e_ms": 22138.79812545183,
|
|
"request_id": 12,
|
|
"tpot_ms": 171.87427427697472,
|
|
"ttft_ms": 310.7652922760442
|
|
},
|
|
{
|
|
"e2e_ms": 22035.248814705075,
|
|
"request_id": 13,
|
|
"tpot_ms": 170.57508935568035,
|
|
"ttft_ms": 372.21246653366967
|
|
},
|
|
{
|
|
"e2e_ms": 21827.324416126074,
|
|
"request_id": 14,
|
|
"tpot_ms": 169.27188182433355,
|
|
"ttft_ms": 329.79542443571484
|
|
},
|
|
{
|
|
"e2e_ms": 21619.373671552865,
|
|
"request_id": 15,
|
|
"tpot_ms": 167.96821838814947,
|
|
"ttft_ms": 287.40993625788303
|
|
},
|
|
{
|
|
"e2e_ms": 21512.77581709026,
|
|
"request_id": 16,
|
|
"tpot_ms": 166.64216905152279,
|
|
"ttft_ms": 349.2203475468658
|
|
},
|
|
{
|
|
"e2e_ms": 21303.255969153783,
|
|
"request_id": 17,
|
|
"tpot_ms": 165.32482604305434,
|
|
"ttft_ms": 307.003061685883
|
|
},
|
|
{
|
|
"e2e_ms": 21196.567333974876,
|
|
"request_id": 18,
|
|
"tpot_ms": 163.99164288866731,
|
|
"ttft_ms": 369.62868711412875
|
|
},
|
|
{
|
|
"e2e_ms": 20986.94051377241,
|
|
"request_id": 19,
|
|
"tpot_ms": 162.67002407488405,
|
|
"ttft_ms": 327.84745626213316
|
|
},
|
|
{
|
|
"e2e_ms": 20777.255965035714,
|
|
"request_id": 20,
|
|
"tpot_ms": 161.34765321373814,
|
|
"ttft_ms": 286.10400689097213
|
|
},
|
|
{
|
|
"e2e_ms": 20670.377044913395,
|
|
"request_id": 21,
|
|
"tpot_ms": 159.9951027007364,
|
|
"ttft_ms": 350.99900191987297
|
|
},
|
|
{
|
|
"e2e_ms": 20460.656782932234,
|
|
"request_id": 22,
|
|
"tpot_ms": 158.67196467076286,
|
|
"ttft_ms": 309.3172697453532
|
|
},
|
|
{
|
|
"e2e_ms": 20353.38846177537,
|
|
"request_id": 23,
|
|
"tpot_ms": 157.31793948495124,
|
|
"ttft_ms": 374.01014718656
|
|
},
|
|
{
|
|
"e2e_ms": 20144.48097023076,
|
|
"request_id": 24,
|
|
"tpot_ms": 155.99981541047092,
|
|
"ttft_ms": 332.5044131009545
|
|
},
|
|
{
|
|
"e2e_ms": 19934.83773528968,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.6729368346084,
|
|
"ttft_ms": 291.37475729441144
|
|
},
|
|
{
|
|
"e2e_ms": 19827.980953909257,
|
|
"request_id": 26,
|
|
"tpot_ms": 153.35264483118294,
|
|
"ttft_ms": 352.1950603490236
|
|
},
|
|
{
|
|
"e2e_ms": 19618.20597824765,
|
|
"request_id": 27,
|
|
"tpot_ms": 152.02412197401114,
|
|
"ttft_ms": 311.1424875482367
|
|
},
|
|
{
|
|
"e2e_ms": 19511.107444230154,
|
|
"request_id": 28,
|
|
"tpot_ms": 150.6975599886974,
|
|
"ttft_ms": 372.5173256655836
|
|
},
|
|
{
|
|
"e2e_ms": 19301.196316545185,
|
|
"request_id": 29,
|
|
"tpot_ms": 149.36537894653213,
|
|
"ttft_ms": 331.7931903356044
|
|
},
|
|
{
|
|
"e2e_ms": 19091.249936104825,
|
|
"request_id": 30,
|
|
"tpot_ms": 148.03267778881258,
|
|
"ttft_ms": 291.0998569256247
|
|
},
|
|
{
|
|
"e2e_ms": 18983.78646717311,
|
|
"request_id": 31,
|
|
"tpot_ms": 146.7011080112165,
|
|
"ttft_ms": 352.7457497486193
|
|
},
|
|
{
|
|
"e2e_ms": 18773.765669374145,
|
|
"request_id": 32,
|
|
"tpot_ms": 145.36784483585674,
|
|
"ttft_ms": 312.0493752203348
|
|
},
|
|
{
|
|
"e2e_ms": 18665.888247551986,
|
|
"request_id": 33,
|
|
"tpot_ms": 144.03124523287858,
|
|
"ttft_ms": 373.9201029764097
|
|
},
|
|
{
|
|
"e2e_ms": 18455.8068747313,
|
|
"request_id": 34,
|
|
"tpot_ms": 142.6968275070543,
|
|
"ttft_ms": 333.30978133540157
|
|
},
|
|
{
|
|
"e2e_ms": 18245.529234152025,
|
|
"request_id": 35,
|
|
"tpot_ms": 141.3602386069844,
|
|
"ttft_ms": 292.7789310650048
|
|
},
|
|
{
|
|
"e2e_ms": 18137.294514438578,
|
|
"request_id": 36,
|
|
"tpot_ms": 140.01821042603632,
|
|
"ttft_ms": 354.98179033196385
|
|
},
|
|
{
|
|
"e2e_ms": 17926.83360591763,
|
|
"request_id": 37,
|
|
"tpot_ms": 138.67967407469433,
|
|
"ttft_ms": 314.5149984314486
|
|
},
|
|
{
|
|
"e2e_ms": 17716.242055541137,
|
|
"request_id": 38,
|
|
"tpot_ms": 137.3398665125213,
|
|
"ttft_ms": 274.0790084509328
|
|
},
|
|
{
|
|
"e2e_ms": 17607.549497877386,
|
|
"request_id": 39,
|
|
"tpot_ms": 135.99225690953574,
|
|
"ttft_ms": 336.53287036634794
|
|
},
|
|
{
|
|
"e2e_ms": 17401.54255835392,
|
|
"request_id": 40,
|
|
"tpot_ms": 134.6880568492608,
|
|
"ttft_ms": 296.1593384978016
|
|
},
|
|
{
|
|
"e2e_ms": 17301.7591255476,
|
|
"request_id": 41,
|
|
"tpot_ms": 133.41188413640484,
|
|
"ttft_ms": 358.4498402241838
|
|
},
|
|
{
|
|
"e2e_ms": 17095.506427630044,
|
|
"request_id": 42,
|
|
"tpot_ms": 132.10522228512968,
|
|
"ttft_ms": 318.1431974185731
|
|
},
|
|
{
|
|
"e2e_ms": 16889.623559688287,
|
|
"request_id": 43,
|
|
"tpot_ms": 130.8012299461049,
|
|
"ttft_ms": 277.86735653296194
|
|
},
|
|
{
|
|
"e2e_ms": 16788.559022297493,
|
|
"request_id": 44,
|
|
"tpot_ms": 129.5126704443812,
|
|
"ttft_ms": 340.449875861081
|
|
},
|
|
{
|
|
"e2e_ms": 16581.727329092846,
|
|
"request_id": 45,
|
|
"tpot_ms": 128.200721970688,
|
|
"ttft_ms": 300.23563881547057
|
|
},
|
|
{
|
|
"e2e_ms": 16478.854197918037,
|
|
"request_id": 46,
|
|
"tpot_ms": 126.8967260219465,
|
|
"ttft_ms": 362.9699931308306
|
|
},
|
|
{
|
|
"e2e_ms": 16270.819161297495,
|
|
"request_id": 47,
|
|
"tpot_ms": 125.57481733363996,
|
|
"ttft_ms": 322.8173599252209
|
|
},
|
|
{
|
|
"e2e_ms": 16062.627892404027,
|
|
"request_id": 48,
|
|
"tpot_ms": 124.25122687214136,
|
|
"ttft_ms": 282.72207964207576
|
|
},
|
|
{
|
|
"e2e_ms": 15958.430745404816,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.91862703441548,
|
|
"ttft_ms": 347.7651120340504
|
|
},
|
|
{
|
|
"e2e_ms": 15749.715539461335,
|
|
"request_id": 50,
|
|
"tpot_ms": 121.58574387256766,
|
|
"ttft_ms": 308.3260676452415
|
|
},
|
|
{
|
|
"e2e_ms": 15645.048986506545,
|
|
"request_id": 51,
|
|
"tpot_ms": 120.24919842858912,
|
|
"ttft_ms": 373.40078607572735
|
|
},
|
|
{
|
|
"e2e_ms": 15435.036319721643,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.8997102377205,
|
|
"ttft_ms": 334.77311953113895
|
|
},
|
|
{
|
|
"e2e_ms": 15224.012292267464,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.54201604221191,
|
|
"ttft_ms": 296.1762549065519
|
|
},
|
|
{
|
|
"e2e_ms": 15109.12675155149,
|
|
"request_id": 54,
|
|
"tpot_ms": 116.11789378026353,
|
|
"ttft_ms": 362.1542414580219
|
|
},
|
|
{
|
|
"e2e_ms": 14893.238736493451,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.72141540015762,
|
|
"ttft_ms": 323.6189806734337
|
|
},
|
|
{
|
|
"e2e_ms": 14675.411853468433,
|
|
"request_id": 56,
|
|
"tpot_ms": 113.30942780834319,
|
|
"ttft_ms": 285.1145218088469
|
|
},
|
|
{
|
|
"e2e_ms": 14549.812898218817,
|
|
"request_id": 57,
|
|
"tpot_ms": 111.79839094375423,
|
|
"ttft_ms": 351.4172483620292
|
|
},
|
|
{
|
|
"e2e_ms": 14329.820655618192,
|
|
"request_id": 58,
|
|
"tpot_ms": 110.36886820693505,
|
|
"ttft_ms": 312.97439333744137
|
|
},
|
|
{
|
|
"e2e_ms": 14108.094791808619,
|
|
"request_id": 59,
|
|
"tpot_ms": 108.92545237461232,
|
|
"ttft_ms": 274.56234023285475
|
|
},
|
|
{
|
|
"e2e_ms": 13975.596499372607,
|
|
"request_id": 60,
|
|
"tpot_ms": 107.35951607044248,
|
|
"ttft_ms": 340.93795842641225
|
|
},
|
|
{
|
|
"e2e_ms": 13753.714210521188,
|
|
"request_id": 61,
|
|
"tpot_ms": 105.91438347527057,
|
|
"ttft_ms": 302.58750916182464
|
|
},
|
|
{
|
|
"e2e_ms": 13622.257898704496,
|
|
"request_id": 62,
|
|
"tpot_ms": 104.35568588071192,
|
|
"ttft_ms": 369.08579185408286
|
|
},
|
|
{
|
|
"e2e_ms": 13397.788527763809,
|
|
"request_id": 63,
|
|
"tpot_ms": 102.88969749082138,
|
|
"ttft_ms": 330.7969464294942
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.1875,
|
|
"passed": 12
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.546875,
|
|
"passed": 35
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.90625,
|
|
"passed": 58
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "96c4e04e8f133e66449a5567fcde568f27af44bd9f5d1cadeade66b771a8f0f3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"loads": [
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 31.752947092056274,
|
|
"offered_request_rate": 0.1,
|
|
"request_metrics_sha256": "b45cd3a473a98bee1127b2315d0c0c59ded580275f6e6d10a3e5a52d85d24de4",
|
|
"request_rate_per_gpu": 0.025,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 11656.879780530655,
|
|
"request_id": 0,
|
|
"tpot_ms": 89.67953789863164,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 11729.945143691864,
|
|
"request_id": 1,
|
|
"tpot_ms": 89.99959080602144,
|
|
"ttft_ms": 299.9971113271407
|
|
},
|
|
{
|
|
"e2e_ms": 11712.730146443742,
|
|
"request_id": 2,
|
|
"tpot_ms": 90.00056214193414,
|
|
"ttft_ms": 282.65875441810806
|
|
},
|
|
{
|
|
"e2e_ms": 11785.795509604945,
|
|
"request_id": 3,
|
|
"tpot_ms": 89.99959080602139,
|
|
"ttft_ms": 355.8474772402285
|
|
},
|
|
{
|
|
"e2e_ms": 11770.95360336775,
|
|
"request_id": 4,
|
|
"tpot_ms": 90.01924789792568,
|
|
"ttft_ms": 338.50912033118874
|
|
},
|
|
{
|
|
"e2e_ms": 11756.111697130556,
|
|
"request_id": 5,
|
|
"tpot_ms": 90.0192478979257,
|
|
"ttft_ms": 323.66721409399446
|
|
},
|
|
{
|
|
"e2e_ms": 11741.269790893028,
|
|
"request_id": 6,
|
|
"tpot_ms": 90.01924789792305,
|
|
"ttft_ms": 308.8253078568002
|
|
},
|
|
{
|
|
"e2e_ms": 11726.427884655508,
|
|
"request_id": 7,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 293.98340161927194
|
|
},
|
|
{
|
|
"e2e_ms": 11709.212887407048,
|
|
"request_id": 8,
|
|
"tpot_ms": 90.00056214193147,
|
|
"ttft_ms": 279.1414953817508
|
|
},
|
|
{
|
|
"e2e_ms": 11782.278250567928,
|
|
"request_id": 9,
|
|
"tpot_ms": 89.99959080601886,
|
|
"ttft_ms": 352.3302182035337
|
|
},
|
|
{
|
|
"e2e_ms": 11767.436344330406,
|
|
"request_id": 10,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 334.9918612941707
|
|
},
|
|
{
|
|
"e2e_ms": 11752.594438092885,
|
|
"request_id": 11,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 320.14995505664956
|
|
},
|
|
{
|
|
"e2e_ms": 11737.752531855705,
|
|
"request_id": 12,
|
|
"tpot_ms": 90.01924789792581,
|
|
"ttft_ms": 305.30804881912843
|
|
},
|
|
{
|
|
"e2e_ms": 11722.91062561817,
|
|
"request_id": 13,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 290.46614258194836
|
|
},
|
|
{
|
|
"e2e_ms": 11705.69562836971,
|
|
"request_id": 14,
|
|
"tpot_ms": 90.00056214193148,
|
|
"ttft_ms": 275.624236344413
|
|
},
|
|
{
|
|
"e2e_ms": 11778.76099153056,
|
|
"request_id": 15,
|
|
"tpot_ms": 89.99959080601862,
|
|
"ttft_ms": 348.8129591661959
|
|
},
|
|
{
|
|
"e2e_ms": 11763.919085293026,
|
|
"request_id": 16,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 331.4746022568045
|
|
},
|
|
{
|
|
"e2e_ms": 11749.07717905549,
|
|
"request_id": 17,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 316.63269601926913
|
|
},
|
|
{
|
|
"e2e_ms": 11734.235272817954,
|
|
"request_id": 18,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 301.7907897817338
|
|
},
|
|
{
|
|
"e2e_ms": 11719.39336658042,
|
|
"request_id": 19,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 286.94888354419845
|
|
},
|
|
{
|
|
"e2e_ms": 11702.178369331961,
|
|
"request_id": 20,
|
|
"tpot_ms": 90.00056214193148,
|
|
"ttft_ms": 272.1069773066631
|
|
},
|
|
{
|
|
"e2e_ms": 11775.24373249281,
|
|
"request_id": 21,
|
|
"tpot_ms": 89.99959080601862,
|
|
"ttft_ms": 345.295700128446
|
|
},
|
|
{
|
|
"e2e_ms": 11760.401826255276,
|
|
"request_id": 22,
|
|
"tpot_ms": 90.01924789792301,
|
|
"ttft_ms": 327.95734321905456
|
|
},
|
|
{
|
|
"e2e_ms": 11745.55992001774,
|
|
"request_id": 23,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 313.1154369815192
|
|
},
|
|
{
|
|
"e2e_ms": 11730.718013780204,
|
|
"request_id": 24,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 298.2735307439839
|
|
},
|
|
{
|
|
"e2e_ms": 11713.50301653041,
|
|
"request_id": 25,
|
|
"tpot_ms": 90.00056214192095,
|
|
"ttft_ms": 283.43162450644854
|
|
},
|
|
{
|
|
"e2e_ms": 11786.568379689925,
|
|
"request_id": 26,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 356.6203473268956
|
|
},
|
|
{
|
|
"e2e_ms": 11771.726473451054,
|
|
"request_id": 27,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 339.28199041616836
|
|
},
|
|
{
|
|
"e2e_ms": 11756.884567212182,
|
|
"request_id": 28,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 324.4400841772972
|
|
},
|
|
{
|
|
"e2e_ms": 11742.042660973311,
|
|
"request_id": 29,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 309.59817793842603
|
|
},
|
|
{
|
|
"e2e_ms": 11727.20075473444,
|
|
"request_id": 30,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 294.7562716995549
|
|
},
|
|
{
|
|
"e2e_ms": 11709.985757484617,
|
|
"request_id": 31,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 279.9143654606837
|
|
},
|
|
{
|
|
"e2e_ms": 11783.051120644132,
|
|
"request_id": 32,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 353.1030882811024
|
|
},
|
|
{
|
|
"e2e_ms": 11768.20921440526,
|
|
"request_id": 33,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 335.7647313703751
|
|
},
|
|
{
|
|
"e2e_ms": 11753.36730816639,
|
|
"request_id": 34,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 320.92282513150394
|
|
},
|
|
{
|
|
"e2e_ms": 11738.525401927518,
|
|
"request_id": 35,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 306.0809188926328
|
|
},
|
|
{
|
|
"e2e_ms": 11723.683495688647,
|
|
"request_id": 36,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 291.2390126537616
|
|
},
|
|
{
|
|
"e2e_ms": 11706.468498438824,
|
|
"request_id": 37,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 276.39710641489046
|
|
},
|
|
{
|
|
"e2e_ms": 11779.533861598338,
|
|
"request_id": 38,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 349.5858292353091
|
|
},
|
|
{
|
|
"e2e_ms": 11764.691955359467,
|
|
"request_id": 39,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 332.24747232458185
|
|
},
|
|
{
|
|
"e2e_ms": 11749.850049120596,
|
|
"request_id": 40,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 317.4055660857107
|
|
},
|
|
{
|
|
"e2e_ms": 11735.008142881725,
|
|
"request_id": 41,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 302.5636598468395
|
|
},
|
|
{
|
|
"e2e_ms": 11720.166236642854,
|
|
"request_id": 42,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 287.72175360796837
|
|
},
|
|
{
|
|
"e2e_ms": 11702.95123939303,
|
|
"request_id": 43,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 272.8798473690972
|
|
},
|
|
{
|
|
"e2e_ms": 11776.016602552545,
|
|
"request_id": 44,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 346.06857018951587
|
|
},
|
|
{
|
|
"e2e_ms": 11761.174696313674,
|
|
"request_id": 45,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 328.7302132787886
|
|
},
|
|
{
|
|
"e2e_ms": 11746.332790074803,
|
|
"request_id": 46,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 313.88830703991744
|
|
},
|
|
{
|
|
"e2e_ms": 11731.490883835932,
|
|
"request_id": 47,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 299.0464008010463
|
|
},
|
|
{
|
|
"e2e_ms": 11716.64897759706,
|
|
"request_id": 48,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 284.2044945621751
|
|
},
|
|
{
|
|
"e2e_ms": 11699.433980347238,
|
|
"request_id": 49,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 269.36258832330395
|
|
},
|
|
{
|
|
"e2e_ms": 11772.499343506752,
|
|
"request_id": 50,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 342.5513111437226
|
|
},
|
|
{
|
|
"e2e_ms": 11757.657437270042,
|
|
"request_id": 51,
|
|
"tpot_ms": 90.0192478979295,
|
|
"ttft_ms": 325.21295423299534
|
|
},
|
|
{
|
|
"e2e_ms": 11742.815531033444,
|
|
"request_id": 52,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 310.3710479967958
|
|
},
|
|
{
|
|
"e2e_ms": 11727.973624796847,
|
|
"request_id": 53,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 295.5291417601984
|
|
},
|
|
{
|
|
"e2e_ms": 11710.758627549352,
|
|
"request_id": 54,
|
|
"tpot_ms": 90.00056214193505,
|
|
"ttft_ms": 280.687235523601
|
|
},
|
|
{
|
|
"e2e_ms": 11783.823990711198,
|
|
"request_id": 55,
|
|
"tpot_ms": 89.99959080602287,
|
|
"ttft_ms": 353.8759583462934
|
|
},
|
|
{
|
|
"e2e_ms": 11768.982084474601,
|
|
"request_id": 56,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 336.5376014379535
|
|
},
|
|
{
|
|
"e2e_ms": 11754.140178238004,
|
|
"request_id": 57,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 321.6956952013561
|
|
},
|
|
{
|
|
"e2e_ms": 11739.298272001406,
|
|
"request_id": 58,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 306.8537889647587
|
|
},
|
|
{
|
|
"e2e_ms": 11724.456365764809,
|
|
"request_id": 59,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 292.01188272816125
|
|
},
|
|
{
|
|
"e2e_ms": 11707.241368517316,
|
|
"request_id": 60,
|
|
"tpot_ms": 90.00056214193505,
|
|
"ttft_ms": 277.16997649156383
|
|
},
|
|
{
|
|
"e2e_ms": 11780.30673167916,
|
|
"request_id": 61,
|
|
"tpot_ms": 89.99959080602287,
|
|
"ttft_ms": 350.3586993142562
|
|
},
|
|
{
|
|
"e2e_ms": 11765.464825442563,
|
|
"request_id": 62,
|
|
"tpot_ms": 90.01924789792635,
|
|
"ttft_ms": 333.0203424059164
|
|
},
|
|
{
|
|
"e2e_ms": 11530.943387303068,
|
|
"request_id": 63,
|
|
"tpot_ms": 88.28948780420275,
|
|
"ttft_ms": 318.17843616931896
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "37e166453683700c9df7abffa278431119205a29c83e92211ffa69d90e1dca0e"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 28.59816026687622,
|
|
"offered_request_rate": 0.2,
|
|
"request_metrics_sha256": "7930ffb53e440b9eefa724912c777842a9c33cefdb46d9014e1c5003223221d2",
|
|
"request_rate_per_gpu": 0.05,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 11975.67900227787,
|
|
"request_id": 0,
|
|
"tpot_ms": 92.18976799112939,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 12131.199338988026,
|
|
"request_id": 1,
|
|
"tpot_ms": 93.2890801728389,
|
|
"ttft_ms": 283.4861570374869
|
|
},
|
|
{
|
|
"e2e_ms": 12196.411822684866,
|
|
"request_id": 2,
|
|
"tpot_ms": 93.31228634935276,
|
|
"ttft_ms": 345.75145631706584
|
|
},
|
|
{
|
|
"e2e_ms": 12171.220586311463,
|
|
"request_id": 3,
|
|
"tpot_ms": 93.31547315047263,
|
|
"ttft_ms": 320.15549620144054
|
|
},
|
|
{
|
|
"e2e_ms": 12145.933482881133,
|
|
"request_id": 4,
|
|
"tpot_ms": 93.31640326087744,
|
|
"ttft_ms": 294.75026874969853
|
|
},
|
|
{
|
|
"e2e_ms": 12120.646379450847,
|
|
"request_id": 5,
|
|
"tpot_ms": 93.31564840216181,
|
|
"ttft_ms": 269.5590323762964
|
|
},
|
|
{
|
|
"e2e_ms": 12185.858863147729,
|
|
"request_id": 6,
|
|
"tpot_ms": 93.31471829175732,
|
|
"ttft_ms": 334.88964009454844
|
|
},
|
|
{
|
|
"e2e_ms": 12160.571759717443,
|
|
"request_id": 7,
|
|
"tpot_ms": 93.31471829175764,
|
|
"ttft_ms": 309.6025366642223
|
|
},
|
|
{
|
|
"e2e_ms": 12135.284656287155,
|
|
"request_id": 8,
|
|
"tpot_ms": 93.31564840216245,
|
|
"ttft_ms": 284.19730921252295
|
|
},
|
|
{
|
|
"e2e_ms": 12200.497139984038,
|
|
"request_id": 9,
|
|
"tpot_ms": 93.31471829175764,
|
|
"ttft_ms": 349.52791693081764
|
|
},
|
|
{
|
|
"e2e_ms": 12175.305903610677,
|
|
"request_id": 10,
|
|
"tpot_ms": 93.3154731504736,
|
|
"ttft_ms": 324.2408135005306
|
|
},
|
|
{
|
|
"e2e_ms": 12150.018800180305,
|
|
"request_id": 11,
|
|
"tpot_ms": 93.31640326087775,
|
|
"ttft_ms": 298.8355860488312
|
|
},
|
|
{
|
|
"e2e_ms": 12124.731696749934,
|
|
"request_id": 12,
|
|
"tpot_ms": 93.31564840216113,
|
|
"ttft_ms": 273.6443496754717
|
|
},
|
|
{
|
|
"e2e_ms": 12189.94418044673,
|
|
"request_id": 13,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 338.97495739375927
|
|
},
|
|
{
|
|
"e2e_ms": 12164.657077016358,
|
|
"request_id": 14,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 313.68785396338694
|
|
},
|
|
{
|
|
"e2e_ms": 12139.369973585985,
|
|
"request_id": 15,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 288.2826265116023
|
|
},
|
|
{
|
|
"e2e_ms": 12204.582457282782,
|
|
"request_id": 16,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 353.6132342298117
|
|
},
|
|
{
|
|
"e2e_ms": 12179.391220909338,
|
|
"request_id": 17,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 328.3261307994394
|
|
},
|
|
{
|
|
"e2e_ms": 12154.104117478966,
|
|
"request_id": 18,
|
|
"tpot_ms": 93.31640326087647,
|
|
"ttft_ms": 302.92090334765476
|
|
},
|
|
{
|
|
"e2e_ms": 12128.817014048593,
|
|
"request_id": 19,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 277.72966697421
|
|
},
|
|
{
|
|
"e2e_ms": 12194.02949774539,
|
|
"request_id": 20,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 343.0602746924194
|
|
},
|
|
{
|
|
"e2e_ms": 12168.742394315017,
|
|
"request_id": 21,
|
|
"tpot_ms": 93.31471829175567,
|
|
"ttft_ms": 317.7731712620471
|
|
},
|
|
{
|
|
"e2e_ms": 12143.455290884645,
|
|
"request_id": 22,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 292.36794381026243
|
|
},
|
|
{
|
|
"e2e_ms": 12208.667774581443,
|
|
"request_id": 23,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 357.69855152847185
|
|
},
|
|
{
|
|
"e2e_ms": 12183.476538207999,
|
|
"request_id": 24,
|
|
"tpot_ms": 93.31547315047165,
|
|
"ttft_ms": 332.4114480980995
|
|
},
|
|
{
|
|
"e2e_ms": 12158.189434777625,
|
|
"request_id": 25,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 307.0062206463149
|
|
},
|
|
{
|
|
"e2e_ms": 12132.902331347253,
|
|
"request_id": 26,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 281.8149842728701
|
|
},
|
|
{
|
|
"e2e_ms": 12198.11481504405,
|
|
"request_id": 27,
|
|
"tpot_ms": 93.31471829175567,
|
|
"ttft_ms": 347.1455919910795
|
|
},
|
|
{
|
|
"e2e_ms": 12172.923578670605,
|
|
"request_id": 28,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 321.8584885607072
|
|
},
|
|
{
|
|
"e2e_ms": 12147.636475240233,
|
|
"request_id": 29,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 296.45326110892256
|
|
},
|
|
{
|
|
"e2e_ms": 12122.349371809862,
|
|
"request_id": 30,
|
|
"tpot_ms": 93.3156484021605,
|
|
"ttft_ms": 271.2620247354778
|
|
},
|
|
{
|
|
"e2e_ms": 12187.561855506658,
|
|
"request_id": 31,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 336.5926324536872
|
|
},
|
|
{
|
|
"e2e_ms": 12162.274752076286,
|
|
"request_id": 32,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 311.3055290233149
|
|
},
|
|
{
|
|
"e2e_ms": 12136.987648645914,
|
|
"request_id": 33,
|
|
"tpot_ms": 93.3156484021605,
|
|
"ttft_ms": 285.90030157153024
|
|
},
|
|
{
|
|
"e2e_ms": 12202.20013234271,
|
|
"request_id": 34,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 351.23090928973966
|
|
},
|
|
{
|
|
"e2e_ms": 12177.008895969266,
|
|
"request_id": 35,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 325.94380585936733
|
|
},
|
|
{
|
|
"e2e_ms": 12151.721792538894,
|
|
"request_id": 36,
|
|
"tpot_ms": 93.31640326087647,
|
|
"ttft_ms": 300.5385784075827
|
|
},
|
|
{
|
|
"e2e_ms": 12126.43468910852,
|
|
"request_id": 37,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 275.3473420341379
|
|
},
|
|
{
|
|
"e2e_ms": 12191.647172805318,
|
|
"request_id": 38,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 340.67794975234733
|
|
},
|
|
{
|
|
"e2e_ms": 12166.360069374947,
|
|
"request_id": 39,
|
|
"tpot_ms": 93.3147182917557,
|
|
"ttft_ms": 315.390846321975
|
|
},
|
|
{
|
|
"e2e_ms": 12141.072965944573,
|
|
"request_id": 40,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 289.98561887019036
|
|
},
|
|
{
|
|
"e2e_ms": 12206.285449641371,
|
|
"request_id": 41,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 355.3162265883998
|
|
},
|
|
{
|
|
"e2e_ms": 12181.094213267927,
|
|
"request_id": 42,
|
|
"tpot_ms": 93.31547315047165,
|
|
"ttft_ms": 330.02912315802746
|
|
},
|
|
{
|
|
"e2e_ms": 12155.807109837553,
|
|
"request_id": 43,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 304.6238957062428
|
|
},
|
|
{
|
|
"e2e_ms": 12130.520006407181,
|
|
"request_id": 44,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 279.43265933279804
|
|
},
|
|
{
|
|
"e2e_ms": 12195.73249010398,
|
|
"request_id": 45,
|
|
"tpot_ms": 93.3147182917557,
|
|
"ttft_ms": 344.76326705100746
|
|
},
|
|
{
|
|
"e2e_ms": 12170.445386673606,
|
|
"request_id": 46,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 319.47616362063513
|
|
},
|
|
{
|
|
"e2e_ms": 12145.158283243234,
|
|
"request_id": 47,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 294.0709361688505
|
|
},
|
|
{
|
|
"e2e_ms": 12210.37076694003,
|
|
"request_id": 48,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 359.4015438870599
|
|
},
|
|
{
|
|
"e2e_ms": 12185.179530566586,
|
|
"request_id": 49,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 334.1144404566876
|
|
},
|
|
{
|
|
"e2e_ms": 12159.892427136241,
|
|
"request_id": 50,
|
|
"tpot_ms": 93.31640326087668,
|
|
"ttft_ms": 308.70921300490295
|
|
},
|
|
{
|
|
"e2e_ms": 12134.605323705899,
|
|
"request_id": 51,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 283.51797663145817
|
|
},
|
|
{
|
|
"e2e_ms": 12199.817807402724,
|
|
"request_id": 52,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 348.848584349696
|
|
},
|
|
{
|
|
"e2e_ms": 12174.626571029308,
|
|
"request_id": 53,
|
|
"tpot_ms": 93.3154731504721,
|
|
"ttft_ms": 323.5614809193521
|
|
},
|
|
{
|
|
"e2e_ms": 12149.339467598964,
|
|
"request_id": 54,
|
|
"tpot_ms": 93.31640326087691,
|
|
"ttft_ms": 298.1562534675959
|
|
},
|
|
{
|
|
"e2e_ms": 12124.05236416862,
|
|
"request_id": 55,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 272.96501709417953
|
|
},
|
|
{
|
|
"e2e_ms": 12189.264847865445,
|
|
"request_id": 56,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 338.2956248124174
|
|
},
|
|
{
|
|
"e2e_ms": 12163.977744435102,
|
|
"request_id": 57,
|
|
"tpot_ms": 93.31471829175614,
|
|
"ttft_ms": 313.00852138207347
|
|
},
|
|
{
|
|
"e2e_ms": 12138.690641004758,
|
|
"request_id": 58,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 287.60329393031725
|
|
},
|
|
{
|
|
"e2e_ms": 12203.903124701583,
|
|
"request_id": 59,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 352.9339016485551
|
|
},
|
|
{
|
|
"e2e_ms": 12178.711888328166,
|
|
"request_id": 60,
|
|
"tpot_ms": 93.31547315047209,
|
|
"ttft_ms": 327.6467982182112
|
|
},
|
|
{
|
|
"e2e_ms": 12153.424784897823,
|
|
"request_id": 61,
|
|
"tpot_ms": 93.31640326087691,
|
|
"ttft_ms": 302.24157076645497
|
|
},
|
|
{
|
|
"e2e_ms": 11949.074413391485,
|
|
"request_id": 62,
|
|
"tpot_ms": 91.90570140943657,
|
|
"ttft_ms": 277.0503343930386
|
|
},
|
|
{
|
|
"e2e_ms": 11702.728382593137,
|
|
"request_id": 63,
|
|
"tpot_ms": 89.45155464946347,
|
|
"ttft_ms": 342.38094211127645
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "e6cebfc858981b2ae4632d2c75f5e8c1f83280e7306351e15c7cd611d2664d80"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 24.987496376037598,
|
|
"offered_request_rate": 0.4,
|
|
"request_metrics_sha256": "e62567aa33ec429331fcd9f24053a0c79db61ef288f7994e6488ae0f0fa79b7e",
|
|
"request_rate_per_gpu": 0.1,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12394.3405299904,
|
|
"request_id": 0,
|
|
"tpot_ms": 95.48631545343278,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 12520.094438712134,
|
|
"request_id": 1,
|
|
"tpot_ms": 96.04733720338449,
|
|
"ttft_ms": 322.08261388230494
|
|
},
|
|
{
|
|
"e2e_ms": 12556.880443424032,
|
|
"request_id": 2,
|
|
"tpot_ms": 96.10807445129032,
|
|
"ttft_ms": 351.1549881101628
|
|
},
|
|
{
|
|
"e2e_ms": 12502.892311945501,
|
|
"request_id": 3,
|
|
"tpot_ms": 96.14618123456218,
|
|
"ttft_ms": 292.32729515610424
|
|
},
|
|
{
|
|
"e2e_ms": 12539.549613984442,
|
|
"request_id": 4,
|
|
"tpot_ms": 96.30977229626299,
|
|
"ttft_ms": 308.2085323590427
|
|
},
|
|
{
|
|
"e2e_ms": 12576.335618696337,
|
|
"request_id": 5,
|
|
"tpot_ms": 96.32399459689755,
|
|
"ttft_ms": 343.188304890349
|
|
},
|
|
{
|
|
"e2e_ms": 12522.347487217807,
|
|
"request_id": 6,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 289.200173411821
|
|
},
|
|
{
|
|
"e2e_ms": 12559.133491929704,
|
|
"request_id": 7,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 325.98617812371697
|
|
},
|
|
{
|
|
"e2e_ms": 12505.145360451173,
|
|
"request_id": 8,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 271.99804664518723
|
|
},
|
|
{
|
|
"e2e_ms": 12541.802662490163,
|
|
"request_id": 9,
|
|
"tpot_ms": 96.32399459689792,
|
|
"ttft_ms": 308.6553486841268
|
|
},
|
|
{
|
|
"e2e_ms": 12578.588667202104,
|
|
"request_id": 10,
|
|
"tpot_ms": 96.32399459689827,
|
|
"ttft_ms": 345.4413533960228
|
|
},
|
|
{
|
|
"e2e_ms": 12524.60053572362,
|
|
"request_id": 11,
|
|
"tpot_ms": 96.32399459689864,
|
|
"ttft_ms": 291.453221917493
|
|
},
|
|
{
|
|
"e2e_ms": 12561.386540435564,
|
|
"request_id": 12,
|
|
"tpot_ms": 96.32399459689903,
|
|
"ttft_ms": 328.239226629389
|
|
},
|
|
{
|
|
"e2e_ms": 12507.39840895708,
|
|
"request_id": 13,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 274.2510951508592
|
|
},
|
|
{
|
|
"e2e_ms": 12544.05571099607,
|
|
"request_id": 14,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 310.90839718984853
|
|
},
|
|
{
|
|
"e2e_ms": 12580.841715708011,
|
|
"request_id": 15,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 347.6944019017907
|
|
},
|
|
{
|
|
"e2e_ms": 12526.853584229528,
|
|
"request_id": 16,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 293.7062704233071
|
|
},
|
|
{
|
|
"e2e_ms": 12563.63958894147,
|
|
"request_id": 17,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 330.49227513524926
|
|
},
|
|
{
|
|
"e2e_ms": 12509.651457462987,
|
|
"request_id": 18,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 276.5041436567657
|
|
},
|
|
{
|
|
"e2e_ms": 12546.308759501975,
|
|
"request_id": 19,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 313.161445695755
|
|
},
|
|
{
|
|
"e2e_ms": 12583.094764213918,
|
|
"request_id": 20,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 349.94745040769715
|
|
},
|
|
{
|
|
"e2e_ms": 12529.106632735506,
|
|
"request_id": 21,
|
|
"tpot_ms": 96.32399459689994,
|
|
"ttft_ms": 295.95931892921357
|
|
},
|
|
{
|
|
"e2e_ms": 12565.892637447447,
|
|
"request_id": 22,
|
|
"tpot_ms": 96.32399459689992,
|
|
"ttft_ms": 332.7453236411557
|
|
},
|
|
{
|
|
"e2e_ms": 12511.904505968956,
|
|
"request_id": 23,
|
|
"tpot_ms": 96.32399459689988,
|
|
"ttft_ms": 278.75719216267214
|
|
},
|
|
{
|
|
"e2e_ms": 12548.561808007931,
|
|
"request_id": 24,
|
|
"tpot_ms": 96.32399459689977,
|
|
"ttft_ms": 315.41449420166146
|
|
},
|
|
{
|
|
"e2e_ms": 12585.347812719874,
|
|
"request_id": 25,
|
|
"tpot_ms": 96.32399459689977,
|
|
"ttft_ms": 352.2004989136036
|
|
},
|
|
{
|
|
"e2e_ms": 12531.359681241383,
|
|
"request_id": 26,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 298.2123674351982
|
|
},
|
|
{
|
|
"e2e_ms": 12568.145685953326,
|
|
"request_id": 27,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 334.99837214714034
|
|
},
|
|
{
|
|
"e2e_ms": 12514.157554474836,
|
|
"request_id": 28,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 281.01024066864966
|
|
},
|
|
{
|
|
"e2e_ms": 12550.81485651381,
|
|
"request_id": 29,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 317.66754270762476
|
|
},
|
|
{
|
|
"e2e_ms": 12587.600861225752,
|
|
"request_id": 30,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 354.4535474195669
|
|
},
|
|
{
|
|
"e2e_ms": 12533.612729747261,
|
|
"request_id": 31,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 300.46541594107623
|
|
},
|
|
{
|
|
"e2e_ms": 12570.398734459204,
|
|
"request_id": 32,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 337.2514206530184
|
|
},
|
|
{
|
|
"e2e_ms": 12516.410602980714,
|
|
"request_id": 33,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 283.2632891745277
|
|
},
|
|
{
|
|
"e2e_ms": 12553.067905019689,
|
|
"request_id": 34,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 319.9205912135028
|
|
},
|
|
{
|
|
"e2e_ms": 12589.85390973163,
|
|
"request_id": 35,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 356.70659592544496
|
|
},
|
|
{
|
|
"e2e_ms": 12535.865778253139,
|
|
"request_id": 36,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 302.7184644469543
|
|
},
|
|
{
|
|
"e2e_ms": 12572.651782965082,
|
|
"request_id": 37,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 339.5044691588964
|
|
},
|
|
{
|
|
"e2e_ms": 12518.663651486591,
|
|
"request_id": 38,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 285.51633768040574
|
|
},
|
|
{
|
|
"e2e_ms": 12555.320953525566,
|
|
"request_id": 39,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 322.17363971938084
|
|
},
|
|
{
|
|
"e2e_ms": 12770.519034100516,
|
|
"request_id": 40,
|
|
"tpot_ms": 97.72881409188341,
|
|
"ttft_ms": 358.959644431323
|
|
},
|
|
{
|
|
"e2e_ms": 12538.070601439798,
|
|
"request_id": 41,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 304.9715129528323
|
|
},
|
|
{
|
|
"e2e_ms": 12574.856606151741,
|
|
"request_id": 42,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 341.75751766477447
|
|
},
|
|
{
|
|
"e2e_ms": 12520.86847467325,
|
|
"request_id": 43,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 287.7693861862838
|
|
},
|
|
{
|
|
"e2e_ms": 12557.654479385192,
|
|
"request_id": 44,
|
|
"tpot_ms": 96.32462827684986,
|
|
"ttft_ms": 324.4266882252589
|
|
},
|
|
{
|
|
"e2e_ms": 12503.666347906701,
|
|
"request_id": 45,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 270.51903410051636
|
|
},
|
|
{
|
|
"e2e_ms": 12540.323649945676,
|
|
"request_id": 46,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 307.17633613949147
|
|
},
|
|
{
|
|
"e2e_ms": 12577.109654657448,
|
|
"request_id": 47,
|
|
"tpot_ms": 96.32399459689775,
|
|
"ttft_ms": 343.9623408514336
|
|
},
|
|
{
|
|
"e2e_ms": 12523.121523178786,
|
|
"request_id": 48,
|
|
"tpot_ms": 96.3239945968964,
|
|
"ttft_ms": 289.97420937294294
|
|
},
|
|
{
|
|
"e2e_ms": 12559.907527890544,
|
|
"request_id": 49,
|
|
"tpot_ms": 96.32399459689495,
|
|
"ttft_ms": 326.7602140848851
|
|
},
|
|
{
|
|
"e2e_ms": 12505.919396411884,
|
|
"request_id": 50,
|
|
"tpot_ms": 96.32399459689363,
|
|
"ttft_ms": 272.7720826063944
|
|
},
|
|
{
|
|
"e2e_ms": 12542.576698450688,
|
|
"request_id": 51,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 309.4293846453695
|
|
},
|
|
{
|
|
"e2e_ms": 12579.362703162446,
|
|
"request_id": 52,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 346.2153893571269
|
|
},
|
|
{
|
|
"e2e_ms": 12525.374571683784,
|
|
"request_id": 53,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 292.2272578784657
|
|
},
|
|
{
|
|
"e2e_ms": 12562.160576395541,
|
|
"request_id": 54,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 329.0132625902231
|
|
},
|
|
{
|
|
"e2e_ms": 12508.17244491688,
|
|
"request_id": 55,
|
|
"tpot_ms": 96.32399459689226,
|
|
"ttft_ms": 275.0251311115619
|
|
},
|
|
{
|
|
"e2e_ms": 12544.829746955686,
|
|
"request_id": 56,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 311.6824331503665
|
|
},
|
|
{
|
|
"e2e_ms": 12581.615751667443,
|
|
"request_id": 57,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 348.4684378621239
|
|
},
|
|
{
|
|
"e2e_ms": 12527.627620188781,
|
|
"request_id": 58,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 294.4803063834627
|
|
},
|
|
{
|
|
"e2e_ms": 12564.413624900539,
|
|
"request_id": 59,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 331.2663110952201
|
|
},
|
|
{
|
|
"e2e_ms": 12314.305135768365,
|
|
"request_id": 60,
|
|
"tpot_ms": 94.77973981221895,
|
|
"ttft_ms": 277.2781796165589
|
|
},
|
|
{
|
|
"e2e_ms": 12170.36562563149,
|
|
"request_id": 61,
|
|
"tpot_ms": 93.35771766910335,
|
|
"ttft_ms": 313.93548165536345
|
|
},
|
|
{
|
|
"e2e_ms": 12023.933572014499,
|
|
"request_id": 62,
|
|
"tpot_ms": 91.91505579249903,
|
|
"ttft_ms": 350.72148636712086
|
|
},
|
|
{
|
|
"e2e_ms": 11724.699298497455,
|
|
"request_id": 63,
|
|
"tpot_ms": 89.98398380794484,
|
|
"ttft_ms": 296.73335488845964
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "021442528b6c4067d63827496b66b6f530adbe9f4b29732b1aa7aab2112ced9f"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 25.64221477508545,
|
|
"offered_request_rate": 0.8,
|
|
"request_metrics_sha256": "b39b2c6a1c6236e30b246303e0b4d66b80a2515899d8c815efc015ef3f70db54",
|
|
"request_rate_per_gpu": 0.2,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 14028.106647728775,
|
|
"request_id": 0,
|
|
"tpot_ms": 108.35061559310502,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 14267.352384844213,
|
|
"request_id": 1,
|
|
"tpot_ms": 109.65331923259451,
|
|
"ttft_ms": 341.3808423047111
|
|
},
|
|
{
|
|
"e2e_ms": 14406.16288598222,
|
|
"request_id": 2,
|
|
"tpot_ms": 110.64064370419715,
|
|
"ttft_ms": 354.8011355491818
|
|
},
|
|
{
|
|
"e2e_ms": 14444.508775243683,
|
|
"request_id": 3,
|
|
"tpot_ms": 111.53949237897281,
|
|
"ttft_ms": 278.99324311413756
|
|
},
|
|
{
|
|
"e2e_ms": 14585.312621821135,
|
|
"request_id": 4,
|
|
"tpot_ms": 112.58765687327464,
|
|
"ttft_ms": 286.6801989152554
|
|
},
|
|
{
|
|
"e2e_ms": 14727.157204966976,
|
|
"request_id": 5,
|
|
"tpot_ms": 113.57630070139629,
|
|
"ttft_ms": 302.9670158896467
|
|
},
|
|
{
|
|
"e2e_ms": 14871.045643448613,
|
|
"request_id": 6,
|
|
"tpot_ms": 114.43037546464966,
|
|
"ttft_ms": 338.38795943810675
|
|
},
|
|
{
|
|
"e2e_ms": 14913.487485283544,
|
|
"request_id": 7,
|
|
"tpot_ms": 115.12950641852738,
|
|
"ttft_ms": 292.0401701305657
|
|
},
|
|
{
|
|
"e2e_ms": 15059.457396901958,
|
|
"request_id": 8,
|
|
"tpot_ms": 115.80285438725795,
|
|
"ttft_ms": 352.4948897201998
|
|
},
|
|
{
|
|
"e2e_ms": 15271.481983406971,
|
|
"request_id": 9,
|
|
"tpot_ms": 117.58757541921905,
|
|
"ttft_ms": 337.85990516615175
|
|
},
|
|
{
|
|
"e2e_ms": 15316.138289012584,
|
|
"request_id": 10,
|
|
"tpot_ms": 117.88145073243837,
|
|
"ttft_ms": 345.1940459929119
|
|
},
|
|
{
|
|
"e2e_ms": 15090.835365768904,
|
|
"request_id": 11,
|
|
"tpot_ms": 116.63565919716638,
|
|
"ttft_ms": 278.1066477287748
|
|
},
|
|
{
|
|
"e2e_ms": 15302.859952273917,
|
|
"request_id": 12,
|
|
"tpot_ms": 118.0348305674148,
|
|
"ttft_ms": 312.43647021223705
|
|
},
|
|
{
|
|
"e2e_ms": 15347.54563377864,
|
|
"request_id": 13,
|
|
"tpot_ms": 118.10044582475892,
|
|
"ttft_ms": 348.7890140342564
|
|
},
|
|
{
|
|
"e2e_ms": 15289.743982068245,
|
|
"request_id": 14,
|
|
"tpot_ms": 118.15013414242924,
|
|
"ttft_ms": 284.6769459797329
|
|
},
|
|
{
|
|
"e2e_ms": 15334.400287673823,
|
|
"request_id": 15,
|
|
"tpot_ms": 118.19982246009941,
|
|
"ttft_ms": 323.022835241197
|
|
},
|
|
{
|
|
"e2e_ms": 15380.492763872971,
|
|
"request_id": 16,
|
|
"tpot_ms": 118.2528557378034,
|
|
"ttft_ms": 362.38008517193873
|
|
},
|
|
{
|
|
"e2e_ms": 15322.691112162565,
|
|
"request_id": 17,
|
|
"tpot_ms": 118.27848723653601,
|
|
"ttft_ms": 301.32323312249196
|
|
},
|
|
{
|
|
"e2e_ms": 15367.376793667254,
|
|
"request_id": 18,
|
|
"tpot_ms": 118.29615526543176,
|
|
"ttft_ms": 343.765074957421
|
|
},
|
|
{
|
|
"e2e_ms": 15309.575141956848,
|
|
"request_id": 19,
|
|
"tpot_ms": 118.30562851819849,
|
|
"ttft_ms": 284.7603201456401
|
|
},
|
|
{
|
|
"e2e_ms": 15354.260823461536,
|
|
"request_id": 20,
|
|
"tpot_ms": 118.30690699483614,
|
|
"ttft_ms": 329.2836351173456
|
|
},
|
|
{
|
|
"e2e_ms": 15296.45917175113,
|
|
"request_id": 21,
|
|
"tpot_ms": 118.3069069948359,
|
|
"ttft_ms": 271.4819834069715
|
|
},
|
|
{
|
|
"e2e_ms": 15341.115477356709,
|
|
"request_id": 22,
|
|
"tpot_ms": 118.30690699483563,
|
|
"ttft_ms": 316.138289012585
|
|
},
|
|
{
|
|
"e2e_ms": 15387.207953555859,
|
|
"request_id": 23,
|
|
"tpot_ms": 118.31926259505173,
|
|
"ttft_ms": 360.6616039842905
|
|
},
|
|
{
|
|
"e2e_ms": 15329.406301845453,
|
|
"request_id": 24,
|
|
"tpot_ms": 118.31926259505147,
|
|
"ttft_ms": 302.8599522739164
|
|
},
|
|
{
|
|
"e2e_ms": 15375.4987780446,
|
|
"request_id": 25,
|
|
"tpot_ms": 118.33033971862962,
|
|
"ttft_ms": 347.5456337786404
|
|
},
|
|
{
|
|
"e2e_ms": 15317.697126334195,
|
|
"request_id": 26,
|
|
"tpot_ms": 118.33033971862953,
|
|
"ttft_ms": 289.74398206824503
|
|
},
|
|
{
|
|
"e2e_ms": 15362.382807838883,
|
|
"request_id": 27,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 334.400287673823
|
|
},
|
|
{
|
|
"e2e_ms": 15304.581156128477,
|
|
"request_id": 28,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 276.59863596341694
|
|
},
|
|
{
|
|
"e2e_ms": 15349.237461734056,
|
|
"request_id": 29,
|
|
"tpot_ms": 118.3192625950511,
|
|
"ttft_ms": 322.6911121625662
|
|
},
|
|
{
|
|
"e2e_ms": 15395.329937933206,
|
|
"request_id": 30,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 367.37679366725473
|
|
},
|
|
{
|
|
"e2e_ms": 15337.5282862228,
|
|
"request_id": 31,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 309.57514195684865
|
|
},
|
|
{
|
|
"e2e_ms": 15383.620762421948,
|
|
"request_id": 32,
|
|
"tpot_ms": 118.34141684220796,
|
|
"ttft_ms": 354.2608234615372
|
|
},
|
|
{
|
|
"e2e_ms": 15325.819110711542,
|
|
"request_id": 33,
|
|
"tpot_ms": 118.34141684220796,
|
|
"ttft_ms": 296.4591717511311
|
|
},
|
|
{
|
|
"e2e_ms": 15370.50479221623,
|
|
"request_id": 34,
|
|
"tpot_ms": 118.34164814850017,
|
|
"ttft_ms": 341.1154773567091
|
|
},
|
|
{
|
|
"e2e_ms": 15312.703140505824,
|
|
"request_id": 35,
|
|
"tpot_ms": 118.34164814850017,
|
|
"ttft_ms": 283.31382564630303
|
|
},
|
|
{
|
|
"e2e_ms": 15357.388822010513,
|
|
"request_id": 36,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 329.4063018454523
|
|
},
|
|
{
|
|
"e2e_ms": 15299.587170300107,
|
|
"request_id": 37,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 271.6046501350462
|
|
},
|
|
{
|
|
"e2e_ms": 15344.243475905685,
|
|
"request_id": 38,
|
|
"tpot_ms": 118.3192625950511,
|
|
"ttft_ms": 317.69712633419545
|
|
},
|
|
{
|
|
"e2e_ms": 15390.335952104835,
|
|
"request_id": 39,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 362.382807838884
|
|
},
|
|
{
|
|
"e2e_ms": 15332.53430039447,
|
|
"request_id": 40,
|
|
"tpot_ms": 118.33033971862987,
|
|
"ttft_ms": 304.5811561284779
|
|
},
|
|
{
|
|
"e2e_ms": 15378.626776593663,
|
|
"request_id": 41,
|
|
"tpot_ms": 118.34164814850084,
|
|
"ttft_ms": 349.2374617340559
|
|
},
|
|
{
|
|
"e2e_ms": 15320.8251248833,
|
|
"request_id": 42,
|
|
"tpot_ms": 118.34164814850119,
|
|
"ttft_ms": 291.4358100236498
|
|
},
|
|
{
|
|
"e2e_ms": 15365.51080638803,
|
|
"request_id": 43,
|
|
"tpot_ms": 118.33057102492307,
|
|
"ttft_ms": 337.52828622279907
|
|
},
|
|
{
|
|
"e2e_ms": 15307.709154677666,
|
|
"request_id": 44,
|
|
"tpot_ms": 118.33057102492342,
|
|
"ttft_ms": 279.726634512393
|
|
},
|
|
{
|
|
"e2e_ms": 15352.365460283294,
|
|
"request_id": 45,
|
|
"tpot_ms": 118.31926259505316,
|
|
"ttft_ms": 325.81911071154224
|
|
},
|
|
{
|
|
"e2e_ms": 15398.457936482486,
|
|
"request_id": 46,
|
|
"tpot_ms": 118.33033971863193,
|
|
"ttft_ms": 370.5047922162308
|
|
},
|
|
{
|
|
"e2e_ms": 15340.656284772123,
|
|
"request_id": 47,
|
|
"tpot_ms": 118.33033971863227,
|
|
"ttft_ms": 312.7031405058247
|
|
},
|
|
{
|
|
"e2e_ms": 15386.748760971315,
|
|
"request_id": 48,
|
|
"tpot_ms": 118.34141684221105,
|
|
"ttft_ms": 357.38882201051325
|
|
},
|
|
{
|
|
"e2e_ms": 15328.947109260951,
|
|
"request_id": 49,
|
|
"tpot_ms": 118.34141684221137,
|
|
"ttft_ms": 299.5871703001072
|
|
},
|
|
{
|
|
"e2e_ms": 15375.039585460143,
|
|
"request_id": 50,
|
|
"tpot_ms": 118.35272527208234,
|
|
"ttft_ms": 344.24347590568516
|
|
},
|
|
{
|
|
"e2e_ms": 15317.23793374978,
|
|
"request_id": 51,
|
|
"tpot_ms": 118.3527252720827,
|
|
"ttft_ms": 286.4418241952791
|
|
},
|
|
{
|
|
"e2e_ms": 15194.4223437212,
|
|
"request_id": 52,
|
|
"tpot_ms": 117.02274049863567,
|
|
"ttft_ms": 332.53430039447096
|
|
},
|
|
{
|
|
"e2e_ms": 14959.152693280232,
|
|
"request_id": 53,
|
|
"tpot_ms": 115.62535468185925,
|
|
"ttft_ms": 274.7326486841075
|
|
},
|
|
{
|
|
"e2e_ms": 14796.032219342294,
|
|
"request_id": 54,
|
|
"tpot_ms": 113.97800861778735,
|
|
"ttft_ms": 320.8251248832994
|
|
},
|
|
{
|
|
"e2e_ms": 14608.764053703837,
|
|
"request_id": 55,
|
|
"tpot_ms": 112.1516003725654,
|
|
"ttft_ms": 365.5108063880306
|
|
},
|
|
{
|
|
"e2e_ms": 14305.495223453576,
|
|
"request_id": 56,
|
|
"tpot_ms": 110.21878794311739,
|
|
"ttft_ms": 307.70915467766713
|
|
},
|
|
{
|
|
"e2e_ms": 14083.450969580696,
|
|
"request_id": 57,
|
|
"tpot_ms": 108.11878353777482,
|
|
"ttft_ms": 352.36546028329485
|
|
},
|
|
{
|
|
"e2e_ms": 13758.528543574443,
|
|
"request_id": 58,
|
|
"tpot_ms": 106.01547035434261,
|
|
"ttft_ms": 294.5638085729314
|
|
},
|
|
{
|
|
"e2e_ms": 13507.044041669204,
|
|
"request_id": 59,
|
|
"tpot_ms": 103.67234454249669,
|
|
"ttft_ms": 340.6562847721233
|
|
},
|
|
{
|
|
"e2e_ms": 13157.052579489218,
|
|
"request_id": 60,
|
|
"tpot_ms": 101.37163737344456,
|
|
"ttft_ms": 282.85463306175984
|
|
},
|
|
{
|
|
"e2e_ms": 12903.847402123616,
|
|
"request_id": 61,
|
|
"tpot_ms": 99.01496293592649,
|
|
"ttft_ms": 328.9471092609517
|
|
},
|
|
{
|
|
"e2e_ms": 12559.065843040158,
|
|
"request_id": 62,
|
|
"tpot_ms": 96.75527862590212,
|
|
"ttft_ms": 271.1454575505883
|
|
},
|
|
{
|
|
"e2e_ms": 12277.402762692504,
|
|
"request_id": 63,
|
|
"tpot_ms": 94.17452621214743,
|
|
"ttft_ms": 317.23793374978015
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "9ed2bf69d3193253248833822a357db5130e6ca29282e93e04cc92961dba468d"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 25.590887784957886,
|
|
"offered_request_rate": 1.2,
|
|
"request_metrics_sha256": "3825f14be2103486ce094e0b335c86deef683645cae8f05ef3d4beb6fb24cf95",
|
|
"request_rate_per_gpu": 0.3,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 15728.871662360909,
|
|
"request_id": 0,
|
|
"tpot_ms": 121.74246610201946,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 15904.548256233129,
|
|
"request_id": 1,
|
|
"tpot_ms": 122.72470152955445,
|
|
"ttft_ms": 318.5111619797136
|
|
},
|
|
{
|
|
"e2e_ms": 16140.50586206045,
|
|
"request_id": 2,
|
|
"tpot_ms": 124.75532551591714,
|
|
"ttft_ms": 296.5795215389744
|
|
},
|
|
{
|
|
"e2e_ms": 16211.825024161684,
|
|
"request_id": 3,
|
|
"tpot_ms": 125.48331552046938,
|
|
"ttft_ms": 275.44395306207383
|
|
},
|
|
{
|
|
"e2e_ms": 16388.84099440166,
|
|
"request_id": 4,
|
|
"tpot_ms": 126.36635826198417,
|
|
"ttft_ms": 340.31349512967114
|
|
},
|
|
{
|
|
"e2e_ms": 16460.2590055204,
|
|
"request_id": 5,
|
|
"tpot_ms": 127.08834011289514,
|
|
"ttft_ms": 320.03981118271605
|
|
},
|
|
{
|
|
"e2e_ms": 16531.726441149392,
|
|
"request_id": 6,
|
|
"tpot_ms": 127.72809453706097,
|
|
"ttft_ms": 310.25843494264916
|
|
},
|
|
{
|
|
"e2e_ms": 16603.193876778383,
|
|
"request_id": 7,
|
|
"tpot_ms": 128.3170301134443,
|
|
"ttft_ms": 306.93105237095784
|
|
},
|
|
{
|
|
"e2e_ms": 16674.731660081485,
|
|
"request_id": 8,
|
|
"tpot_ms": 128.84921275092722,
|
|
"ttft_ms": 310.8816407137276
|
|
},
|
|
{
|
|
"e2e_ms": 16746.318867894835,
|
|
"request_id": 9,
|
|
"tpot_ms": 129.28898520065795,
|
|
"ttft_ms": 326.61774741127834
|
|
},
|
|
{
|
|
"e2e_ms": 16982.978169021266,
|
|
"request_id": 10,
|
|
"tpot_ms": 130.92468009104587,
|
|
"ttft_ms": 355.5437974584379
|
|
},
|
|
{
|
|
"e2e_ms": 16948.91870099869,
|
|
"request_id": 11,
|
|
"tpot_ms": 131.099598546656,
|
|
"ttft_ms": 299.26968557337295
|
|
},
|
|
{
|
|
"e2e_ms": 17020.55604111586,
|
|
"request_id": 12,
|
|
"tpot_ms": 131.2589874619586,
|
|
"ttft_ms": 350.6646334471153
|
|
},
|
|
{
|
|
"e2e_ms": 16986.496573094286,
|
|
"request_id": 13,
|
|
"tpot_ms": 131.335760528223,
|
|
"ttft_ms": 306.854986009963
|
|
},
|
|
{
|
|
"e2e_ms": 17059.03976122884,
|
|
"request_id": 14,
|
|
"tpot_ms": 131.4398579526774,
|
|
"ttft_ms": 366.1778012388126
|
|
},
|
|
{
|
|
"e2e_ms": 17024.98029320726,
|
|
"request_id": 15,
|
|
"tpot_ms": 131.50690676411472,
|
|
"ttft_ms": 323.6031341646921
|
|
},
|
|
{
|
|
"e2e_ms": 16990.920825185683,
|
|
"request_id": 16,
|
|
"tpot_ms": 131.56847300071718,
|
|
"ttft_ms": 281.7247540946006
|
|
},
|
|
{
|
|
"e2e_ms": 17063.464013320234,
|
|
"request_id": 17,
|
|
"tpot_ms": 131.63886355765484,
|
|
"ttft_ms": 345.32834149806746
|
|
},
|
|
{
|
|
"e2e_ms": 17029.40454529866,
|
|
"request_id": 18,
|
|
"tpot_ms": 131.67812927130495,
|
|
"ttft_ms": 306.28212784293396
|
|
},
|
|
{
|
|
"e2e_ms": 16995.3450772771,
|
|
"request_id": 19,
|
|
"tpot_ms": 131.6904025418972,
|
|
"ttft_ms": 270.66395445615575
|
|
},
|
|
{
|
|
"e2e_ms": 17067.888265411675,
|
|
"request_id": 20,
|
|
"tpot_ms": 131.7034495326512,
|
|
"ttft_ms": 341.5501747649721
|
|
},
|
|
{
|
|
"e2e_ms": 17033.828797390113,
|
|
"request_id": 21,
|
|
"tpot_ms": 131.70595487135952,
|
|
"ttft_ms": 307.17252872745246
|
|
},
|
|
{
|
|
"e2e_ms": 16999.769329368555,
|
|
"request_id": 22,
|
|
"tpot_ms": 131.7084602100679,
|
|
"ttft_ms": 272.79488268993293
|
|
},
|
|
{
|
|
"e2e_ms": 17072.312517503127,
|
|
"request_id": 23,
|
|
"tpot_ms": 131.71809821033827,
|
|
"ttft_ms": 344.1140447901674
|
|
},
|
|
{
|
|
"e2e_ms": 17038.25304948157,
|
|
"request_id": 24,
|
|
"tpot_ms": 131.71982521031822,
|
|
"ttft_ms": 309.83524777115434
|
|
},
|
|
{
|
|
"e2e_ms": 17004.193581460007,
|
|
"request_id": 25,
|
|
"tpot_ms": 131.72116304093396,
|
|
"ttft_ms": 275.60587526139454
|
|
},
|
|
{
|
|
"e2e_ms": 17076.736769594583,
|
|
"request_id": 26,
|
|
"tpot_ms": 131.72963353311178,
|
|
"ttft_ms": 347.07331088938884
|
|
},
|
|
{
|
|
"e2e_ms": 17042.67730157302,
|
|
"request_id": 27,
|
|
"tpot_ms": 131.7305821943633,
|
|
"ttft_ms": 312.8933628888824
|
|
},
|
|
{
|
|
"e2e_ms": 17008.617833551463,
|
|
"request_id": 28,
|
|
"tpot_ms": 131.7309769369132,
|
|
"ttft_ms": 278.783762563485
|
|
},
|
|
{
|
|
"e2e_ms": 17081.161021686035,
|
|
"request_id": 29,
|
|
"tpot_ms": 131.73850434102513,
|
|
"ttft_ms": 350.37097037584175
|
|
},
|
|
{
|
|
"e2e_ms": 17047.986478516996,
|
|
"request_id": 30,
|
|
"tpot_ms": 131.74547225324986,
|
|
"ttft_ms": 316.3115023542638
|
|
},
|
|
{
|
|
"e2e_ms": 17013.927010495438,
|
|
"request_id": 31,
|
|
"tpot_ms": 131.74547225325,
|
|
"ttft_ms": 282.25203433268575
|
|
},
|
|
{
|
|
"e2e_ms": 17086.47019863001,
|
|
"request_id": 32,
|
|
"tpot_ms": 131.75260491481217,
|
|
"ttft_ms": 353.8893744488618
|
|
},
|
|
{
|
|
"e2e_ms": 17053.29565546097,
|
|
"request_id": 33,
|
|
"tpot_ms": 131.7595728270369,
|
|
"ttft_ms": 319.82990642728385
|
|
},
|
|
{
|
|
"e2e_ms": 17019.236187439412,
|
|
"request_id": 34,
|
|
"tpot_ms": 131.75957282703703,
|
|
"ttft_ms": 285.7704384057058
|
|
},
|
|
{
|
|
"e2e_ms": 17091.779375573984,
|
|
"request_id": 35,
|
|
"tpot_ms": 131.7595728270372,
|
|
"ttft_ms": 358.31362654026225
|
|
},
|
|
{
|
|
"e2e_ms": 17058.604832404944,
|
|
"request_id": 36,
|
|
"tpot_ms": 131.7665407392619,
|
|
"ttft_ms": 324.2541585186842
|
|
},
|
|
{
|
|
"e2e_ms": 17024.545364383386,
|
|
"request_id": 37,
|
|
"tpot_ms": 131.76654073926207,
|
|
"ttft_ms": 290.1946904971062
|
|
},
|
|
{
|
|
"e2e_ms": 17097.08855251796,
|
|
"request_id": 38,
|
|
"tpot_ms": 131.76654073926218,
|
|
"ttft_ms": 362.7378786316626
|
|
},
|
|
{
|
|
"e2e_ms": 17063.934932514778,
|
|
"request_id": 39,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 328.6784106101024
|
|
},
|
|
{
|
|
"e2e_ms": 17029.875464493216,
|
|
"request_id": 40,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 294.61894258854215
|
|
},
|
|
{
|
|
"e2e_ms": 17102.41865262779,
|
|
"request_id": 41,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 367.1621307231163
|
|
},
|
|
{
|
|
"e2e_ms": 17069.265032624608,
|
|
"request_id": 42,
|
|
"tpot_ms": 131.78080606238623,
|
|
"ttft_ms": 333.1026627015561
|
|
},
|
|
{
|
|
"e2e_ms": 17035.205564603046,
|
|
"request_id": 43,
|
|
"tpot_ms": 131.78080606238623,
|
|
"ttft_ms": 299.04319467999585
|
|
},
|
|
{
|
|
"e2e_ms": 16941.812657737883,
|
|
"request_id": 44,
|
|
"tpot_ms": 130.47422263719145,
|
|
"ttft_ms": 371.58638281457
|
|
},
|
|
{
|
|
"e2e_ms": 16742.48916517699,
|
|
"request_id": 45,
|
|
"tpot_ms": 129.1729311053857,
|
|
"ttft_ms": 337.5269147930098
|
|
},
|
|
{
|
|
"e2e_ms": 16538.03643831958,
|
|
"request_id": 46,
|
|
"tpot_ms": 127.83125190195379,
|
|
"ttft_ms": 303.46744677144954
|
|
},
|
|
{
|
|
"e2e_ms": 16435.957848641778,
|
|
"request_id": 47,
|
|
"tpot_ms": 126.45627727350987,
|
|
"ttft_ms": 376.0106349060237
|
|
},
|
|
{
|
|
"e2e_ms": 16228.476901947985,
|
|
"request_id": 48,
|
|
"tpot_ms": 125.09075381939779,
|
|
"ttft_ms": 341.9511668844635
|
|
},
|
|
{
|
|
"e2e_ms": 16020.052127617348,
|
|
"request_id": 49,
|
|
"tpot_ms": 123.7177986516098,
|
|
"ttft_ms": 307.89169886290324
|
|
},
|
|
{
|
|
"e2e_ms": 15809.427558622445,
|
|
"request_id": 50,
|
|
"tpot_ms": 122.32055435377619,
|
|
"ttft_ms": 274.71715569286914
|
|
},
|
|
{
|
|
"e2e_ms": 15703.511289947706,
|
|
"request_id": 51,
|
|
"tpot_ms": 120.91536178046667,
|
|
"ttft_ms": 347.26034382843807
|
|
},
|
|
{
|
|
"e2e_ms": 15485.101955905271,
|
|
"request_id": 52,
|
|
"tpot_ms": 119.46378803227081,
|
|
"ttft_ms": 313.20087580687783
|
|
},
|
|
{
|
|
"e2e_ms": 15260.624457846197,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.95746555282956,
|
|
"ttft_ms": 280.02633263684373
|
|
},
|
|
{
|
|
"e2e_ms": 15118.941732007286,
|
|
"request_id": 54,
|
|
"tpot_ms": 116.27064733255806,
|
|
"ttft_ms": 352.56952077241266
|
|
},
|
|
{
|
|
"e2e_ms": 14865.280308326035,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.54149807539514,
|
|
"ttft_ms": 318.5100527508524
|
|
},
|
|
{
|
|
"e2e_ms": 14599.985676841925,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.71378084457564,
|
|
"ttft_ms": 285.3355095808183
|
|
},
|
|
{
|
|
"e2e_ms": 14420.806000135271,
|
|
"request_id": 57,
|
|
"tpot_ms": 110.73171104266838,
|
|
"ttft_ms": 357.87869771638725
|
|
},
|
|
{
|
|
"e2e_ms": 14142.519211198518,
|
|
"request_id": 58,
|
|
"tpot_ms": 108.80866127168261,
|
|
"ttft_ms": 323.819229694827
|
|
},
|
|
{
|
|
"e2e_ms": 13853.830695007076,
|
|
"request_id": 59,
|
|
"tpot_ms": 106.79657547493251,
|
|
"ttft_ms": 290.6656096906488
|
|
},
|
|
{
|
|
"e2e_ms": 13650.503338148035,
|
|
"request_id": 60,
|
|
"tpot_ms": 104.62436645922692,
|
|
"ttft_ms": 363.20879782621773
|
|
},
|
|
{
|
|
"e2e_ms": 13360.876271706531,
|
|
"request_id": 61,
|
|
"tpot_ms": 102.61202316458169,
|
|
"ttft_ms": 329.1493298046575
|
|
},
|
|
{
|
|
"e2e_ms": 13070.674002922458,
|
|
"request_id": 62,
|
|
"tpot_ms": 100.58801805607857,
|
|
"ttft_ms": 295.9957098004793
|
|
},
|
|
{
|
|
"e2e_ms": 12853.555073004585,
|
|
"request_id": 63,
|
|
"tpot_ms": 98.30721397691761,
|
|
"ttft_ms": 368.5388979360482
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.1875,
|
|
"passed": 12
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "3e07b28ac20428d11b8d83064d68561f03804f8adf37111de43398fdd8d94138"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 28.949723482131958,
|
|
"offered_request_rate": 1.6,
|
|
"request_metrics_sha256": "b7d73a1cbef8ca659f753135a4944cb3023d3a2424a19a741ae2198f39c2d172",
|
|
"request_rate_per_gpu": 0.4,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 17513.570713343794,
|
|
"request_id": 0,
|
|
"tpot_ms": 135.79521453495556,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 17840.282304762335,
|
|
"request_id": 1,
|
|
"tpot_ms": 137.71064841138914,
|
|
"ttft_ms": 351.0299565159147
|
|
},
|
|
{
|
|
"e2e_ms": 17895.63618825912,
|
|
"request_id": 2,
|
|
"tpot_ms": 138.10245652748367,
|
|
"ttft_ms": 356.6242092686922
|
|
},
|
|
{
|
|
"e2e_ms": 17848.77057711112,
|
|
"request_id": 3,
|
|
"tpot_ms": 138.39719968518088,
|
|
"ttft_ms": 272.32621709314975
|
|
},
|
|
{
|
|
"e2e_ms": 17904.1244606079,
|
|
"request_id": 4,
|
|
"tpot_ms": 138.80479077491768,
|
|
"ttft_ms": 275.9160321933569
|
|
},
|
|
{
|
|
"e2e_ms": 17959.478344104686,
|
|
"request_id": 5,
|
|
"tpot_ms": 139.18735062645055,
|
|
"ttft_ms": 282.6848145454632
|
|
},
|
|
{
|
|
"e2e_ms": 18014.921946878916,
|
|
"request_id": 6,
|
|
"tpot_ms": 139.5152186600215,
|
|
"ttft_ms": 296.48917705618504
|
|
},
|
|
{
|
|
"e2e_ms": 18070.36554965315,
|
|
"request_id": 7,
|
|
"tpot_ms": 139.8086106441042,
|
|
"ttft_ms": 314.671997851919
|
|
},
|
|
{
|
|
"e2e_ms": 18125.898871704834,
|
|
"request_id": 8,
|
|
"tpot_ms": 140.0646556798527,
|
|
"ttft_ms": 337.68760036353916
|
|
},
|
|
{
|
|
"e2e_ms": 18079.30241838919,
|
|
"request_id": 9,
|
|
"tpot_ms": 140.21470125635153,
|
|
"ttft_ms": 272.0353588325457
|
|
},
|
|
{
|
|
"e2e_ms": 18134.83860521682,
|
|
"request_id": 10,
|
|
"tpot_ms": 140.33922237580643,
|
|
"ttft_ms": 311.75736348940484
|
|
},
|
|
{
|
|
"e2e_ms": 18190.4645113219,
|
|
"request_id": 11,
|
|
"tpot_ms": 140.37869703173442,
|
|
"ttft_ms": 362.36998829163
|
|
},
|
|
{
|
|
"e2e_ms": 18143.870922782207,
|
|
"request_id": 12,
|
|
"tpot_ms": 140.38698051363622,
|
|
"ttft_ms": 314.7243975504068
|
|
},
|
|
{
|
|
"e2e_ms": 18097.274469466563,
|
|
"request_id": 13,
|
|
"tpot_ms": 140.36074521133403,
|
|
"ttft_ms": 271.4598276271438
|
|
},
|
|
{
|
|
"e2e_ms": 18152.810656294194,
|
|
"request_id": 14,
|
|
"tpot_ms": 140.32455966134754,
|
|
"ttft_ms": 331.5915793030584
|
|
},
|
|
{
|
|
"e2e_ms": 18106.217067754496,
|
|
"request_id": 15,
|
|
"tpot_ms": 140.29231877139563,
|
|
"ttft_ms": 289.0925837872533
|
|
},
|
|
{
|
|
"e2e_ms": 18161.753254582127,
|
|
"request_id": 16,
|
|
"tpot_ms": 140.24009333090902,
|
|
"ttft_ms": 351.2614015566804
|
|
},
|
|
{
|
|
"e2e_ms": 18115.15966604243,
|
|
"request_id": 17,
|
|
"tpot_ms": 140.19912163029534,
|
|
"ttft_ms": 309.87121899492286
|
|
},
|
|
{
|
|
"e2e_ms": 18170.69585287006,
|
|
"request_id": 18,
|
|
"tpot_ms": 140.12856604535176,
|
|
"ttft_ms": 374.3679651103839
|
|
},
|
|
{
|
|
"e2e_ms": 18124.102264330366,
|
|
"request_id": 19,
|
|
"tpot_ms": 140.05569589908265,
|
|
"ttft_ms": 337.0288851468697
|
|
},
|
|
{
|
|
"e2e_ms": 18077.42933293013,
|
|
"request_id": 20,
|
|
"tpot_ms": 139.98073600581233,
|
|
"ttft_ms": 299.87586019196044
|
|
},
|
|
{
|
|
"e2e_ms": 18132.96551975776,
|
|
"request_id": 21,
|
|
"tpot_ms": 139.8495015438229,
|
|
"ttft_ms": 372.0788236922541
|
|
},
|
|
{
|
|
"e2e_ms": 18086.369066442116,
|
|
"request_id": 22,
|
|
"tpot_ms": 139.76199603449132,
|
|
"ttft_ms": 336.5955700617178
|
|
},
|
|
{
|
|
"e2e_ms": 18039.682893849018,
|
|
"request_id": 23,
|
|
"tpot_ms": 139.67091992877667,
|
|
"ttft_ms": 301.4760628943822
|
|
},
|
|
{
|
|
"e2e_ms": 18095.21908067664,
|
|
"request_id": 24,
|
|
"tpot_ms": 139.54200325642663,
|
|
"ttft_ms": 373.38466711045726
|
|
},
|
|
{
|
|
"e2e_ms": 18048.532908083544,
|
|
"request_id": 25,
|
|
"tpot_ms": 139.5517742330953,
|
|
"ttft_ms": 325.4575804804407
|
|
},
|
|
{
|
|
"e2e_ms": 18001.846735490446,
|
|
"request_id": 26,
|
|
"tpot_ms": 139.5610841122856,
|
|
"ttft_ms": 277.589053230173
|
|
},
|
|
{
|
|
"e2e_ms": 18057.290338264673,
|
|
"request_id": 27,
|
|
"tpot_ms": 139.56778701592805,
|
|
"ttft_ms": 332.18138724181046
|
|
},
|
|
{
|
|
"e2e_ms": 18010.60416567157,
|
|
"request_id": 28,
|
|
"tpot_ms": 139.57154350494935,
|
|
"ttft_ms": 285.01814054300564
|
|
},
|
|
{
|
|
"e2e_ms": 18066.06100963866,
|
|
"request_id": 29,
|
|
"tpot_ms": 139.5730606683175,
|
|
"ttft_ms": 340.2823047623365
|
|
},
|
|
{
|
|
"e2e_ms": 18019.37483704556,
|
|
"request_id": 30,
|
|
"tpot_ms": 139.57447357032456,
|
|
"ttft_ms": 293.41669361433986
|
|
},
|
|
{
|
|
"e2e_ms": 18074.831681012645,
|
|
"request_id": 31,
|
|
"tpot_ms": 139.57528428268915,
|
|
"ttft_ms": 348.7705771111216
|
|
},
|
|
{
|
|
"e2e_ms": 18028.145508419548,
|
|
"request_id": 32,
|
|
"tpot_ms": 139.57669718469626,
|
|
"ttft_ms": 301.904965963125
|
|
},
|
|
{
|
|
"e2e_ms": 18083.678830471228,
|
|
"request_id": 33,
|
|
"tpot_ms": 139.5781100867033,
|
|
"ttft_ms": 357.25884945990674
|
|
},
|
|
{
|
|
"e2e_ms": 18036.99265787813,
|
|
"request_id": 34,
|
|
"tpot_ms": 139.57881653770684,
|
|
"ttft_ms": 310.48295758936104
|
|
},
|
|
{
|
|
"e2e_ms": 18092.528844705754,
|
|
"request_id": 35,
|
|
"tpot_ms": 139.57954554600127,
|
|
"ttft_ms": 365.9265603635937
|
|
},
|
|
{
|
|
"e2e_ms": 17876.05367572695,
|
|
"request_id": 36,
|
|
"tpot_ms": 138.24262431461773,
|
|
"ttft_ms": 319.240387770499
|
|
},
|
|
{
|
|
"e2e_ms": 17659.21004164318,
|
|
"request_id": 37,
|
|
"tpot_ms": 136.90209533219152,
|
|
"ttft_ms": 272.64393445485524
|
|
},
|
|
{
|
|
"e2e_ms": 17543.75228976072,
|
|
"request_id": 38,
|
|
"tpot_ms": 135.55570892326125,
|
|
"ttft_ms": 328.1772565065388
|
|
},
|
|
{
|
|
"e2e_ms": 17326.367174433202,
|
|
"request_id": 39,
|
|
"tpot_ms": 134.21089375170362,
|
|
"ttft_ms": 281.5836679668422
|
|
},
|
|
{
|
|
"e2e_ms": 17233.832476815856,
|
|
"request_id": 40,
|
|
"tpot_ms": 133.04427482475538,
|
|
"ttft_ms": 337.20957407192384
|
|
},
|
|
{
|
|
"e2e_ms": 17034.265611203216,
|
|
"request_id": 41,
|
|
"tpot_ms": 131.83976083205505,
|
|
"ttft_ms": 290.6159855322272
|
|
},
|
|
{
|
|
"e2e_ms": 16940.502121615402,
|
|
"request_id": 42,
|
|
"tpot_ms": 130.66419538607474,
|
|
"ttft_ms": 346.14930758391085
|
|
},
|
|
{
|
|
"e2e_ms": 16741.97064984836,
|
|
"request_id": 43,
|
|
"tpot_ms": 129.46783410082003,
|
|
"ttft_ms": 299.5557190442142
|
|
},
|
|
{
|
|
"e2e_ms": 16645.559306371404,
|
|
"request_id": 44,
|
|
"tpot_ms": 128.27139685432724,
|
|
"ttft_ms": 355.0919058718449
|
|
},
|
|
{
|
|
"e2e_ms": 16443.232533552808,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.04515130882409,
|
|
"ttft_ms": 308.4983173321483
|
|
},
|
|
{
|
|
"e2e_ms": 16342.299705615802,
|
|
"request_id": 46,
|
|
"tpot_ms": 125.81311182248837,
|
|
"ttft_ms": 364.034504159779
|
|
},
|
|
{
|
|
"e2e_ms": 16135.159559133626,
|
|
"request_id": 47,
|
|
"tpot_ms": 124.54896569695704,
|
|
"ttft_ms": 317.44091562008236
|
|
},
|
|
{
|
|
"e2e_ms": 16031.7032146663,
|
|
"request_id": 48,
|
|
"tpot_ms": 123.29695174036006,
|
|
"ttft_ms": 372.9903436405735
|
|
},
|
|
{
|
|
"e2e_ms": 15823.308920667878,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.0236594458299,
|
|
"ttft_ms": 326.3041710474788
|
|
},
|
|
{
|
|
"e2e_ms": 15613.44809689394,
|
|
"request_id": 50,
|
|
"tpot_ms": 120.73809066445794,
|
|
"ttft_ms": 279.71058250778214
|
|
},
|
|
{
|
|
"e2e_ms": 15508.031714506984,
|
|
"request_id": 51,
|
|
"tpot_ms": 119.47077015706711,
|
|
"ttft_ms": 335.2439045594622
|
|
},
|
|
{
|
|
"e2e_ms": 15292.981047367362,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.14506547559841,
|
|
"ttft_ms": 288.5577319663639
|
|
},
|
|
{
|
|
"e2e_ms": 15175.360910096466,
|
|
"request_id": 53,
|
|
"tpot_ms": 116.78162985277542,
|
|
"ttft_ms": 344.09391879398754
|
|
},
|
|
{
|
|
"e2e_ms": 14945.589828664517,
|
|
"request_id": 54,
|
|
"tpot_ms": 115.34001639735139,
|
|
"ttft_ms": 297.4077462008893
|
|
},
|
|
{
|
|
"e2e_ms": 14803.64975337431,
|
|
"request_id": 55,
|
|
"tpot_ms": 113.78581420786767,
|
|
"ttft_ms": 352.85134897511483
|
|
},
|
|
{
|
|
"e2e_ms": 14557.342221274233,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.21399247946627,
|
|
"ttft_ms": 306.16517638201657
|
|
},
|
|
{
|
|
"e2e_ms": 14399.594833150197,
|
|
"request_id": 57,
|
|
"tpot_ms": 110.5352189984338,
|
|
"ttft_ms": 361.6220203491025
|
|
},
|
|
{
|
|
"e2e_ms": 14144.625862747695,
|
|
"request_id": 58,
|
|
"tpot_ms": 108.89519696843851,
|
|
"ttft_ms": 314.93584775600425
|
|
},
|
|
{
|
|
"e2e_ms": 13973.496543699825,
|
|
"request_id": 59,
|
|
"tpot_ms": 107.1110539525727,
|
|
"ttft_ms": 370.3926917230902
|
|
},
|
|
{
|
|
"e2e_ms": 13708.4999588278,
|
|
"request_id": 60,
|
|
"tpot_ms": 105.3920743283292,
|
|
"ttft_ms": 323.70651912999193
|
|
},
|
|
{
|
|
"e2e_ms": 13445.970803422128,
|
|
"request_id": 61,
|
|
"tpot_ms": 103.6925232825609,
|
|
"ttft_ms": 277.02034653689367
|
|
},
|
|
{
|
|
"e2e_ms": 13273.580023880399,
|
|
"request_id": 62,
|
|
"tpot_ms": 101.89784531725846,
|
|
"ttft_ms": 332.5536685885737
|
|
},
|
|
{
|
|
"e2e_ms": 13000.702540117643,
|
|
"request_id": 63,
|
|
"tpot_ms": 100.1168113710407,
|
|
"ttft_ms": 285.86749599547545
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.203125,
|
|
"passed": 13
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "850087527f8ec5b333ae4bb00c5837c85f7dafca77de3d8c80820e071da8dfbb"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 41.7739622592926,
|
|
"offered_request_rate": 2.4,
|
|
"request_metrics_sha256": "3b42135fbabbd54647284962ccc9fdf0b900fbc0b6ed0bdecf03477d5c409d18",
|
|
"request_rate_per_gpu": 0.6,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 21942.173036302996,
|
|
"request_id": 0,
|
|
"tpot_ms": 170.66609896770518,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 22008.341136862466,
|
|
"request_id": 1,
|
|
"tpot_ms": 170.96613901739173,
|
|
"ttft_ms": 295.64148165371654
|
|
},
|
|
{
|
|
"e2e_ms": 22074.54102422534,
|
|
"request_id": 2,
|
|
"tpot_ms": 171.22660942224667,
|
|
"ttft_ms": 328.7616276000109
|
|
},
|
|
{
|
|
"e2e_ms": 22036.227907899276,
|
|
"request_id": 3,
|
|
"tpot_ms": 171.37344968894675,
|
|
"ttft_ms": 271.7997974030406
|
|
},
|
|
{
|
|
"e2e_ms": 22102.42779526115,
|
|
"request_id": 4,
|
|
"tpot_ms": 171.64131409153669,
|
|
"ttft_ms": 303.98090563598765
|
|
},
|
|
{
|
|
"e2e_ms": 22168.65946942643,
|
|
"request_id": 5,
|
|
"tpot_ms": 171.89659303259273,
|
|
"ttft_ms": 337.79215428715537
|
|
},
|
|
{
|
|
"e2e_ms": 22130.37813990278,
|
|
"request_id": 6,
|
|
"tpot_ms": 172.0281077294899,
|
|
"ttft_ms": 282.80845825756097
|
|
},
|
|
{
|
|
"e2e_ms": 22365.99994575454,
|
|
"request_id": 7,
|
|
"tpot_ms": 173.57085404199856,
|
|
"ttft_ms": 322.5014824207202
|
|
},
|
|
{
|
|
"e2e_ms": 22158.3592864644,
|
|
"request_id": 8,
|
|
"tpot_ms": 172.3496956452424,
|
|
"ttft_ms": 269.9479395186155
|
|
},
|
|
{
|
|
"e2e_ms": 22393.981092316164,
|
|
"request_id": 9,
|
|
"tpot_ms": 173.8419366523574,
|
|
"ttft_ms": 316.0551374667744
|
|
},
|
|
{
|
|
"e2e_ms": 22460.243568400445,
|
|
"request_id": 10,
|
|
"tpot_ms": 173.96590922572528,
|
|
"ttft_ms": 366.57309673333316
|
|
},
|
|
{
|
|
"e2e_ms": 22252.602909110316,
|
|
"request_id": 11,
|
|
"tpot_ms": 172.6883751510306,
|
|
"ttft_ms": 321.1792649294285
|
|
},
|
|
{
|
|
"e2e_ms": 22044.894176229613,
|
|
"request_id": 12,
|
|
"tpot_ms": 171.40112664047544,
|
|
"ttft_ms": 276.9510928892318
|
|
},
|
|
{
|
|
"e2e_ms": 21941.653543442477,
|
|
"request_id": 13,
|
|
"tpot_ms": 170.1065182287643,
|
|
"ttft_ms": 338.1257283894126
|
|
},
|
|
{
|
|
"e2e_ms": 21734.138079618475,
|
|
"request_id": 14,
|
|
"tpot_ms": 168.80971544527205,
|
|
"ttft_ms": 295.30421806892537
|
|
},
|
|
{
|
|
"e2e_ms": 21631.119179231846,
|
|
"request_id": 15,
|
|
"tpot_ms": 167.51267286797687,
|
|
"ttft_ms": 357.0097249987798
|
|
},
|
|
{
|
|
"e2e_ms": 21420.354658102242,
|
|
"request_id": 16,
|
|
"tpot_ms": 166.18662353135022,
|
|
"ttft_ms": 314.65346962076256
|
|
},
|
|
{
|
|
"e2e_ms": 21209.648295563282,
|
|
"request_id": 17,
|
|
"tpot_ms": 164.85776701341018,
|
|
"ttft_ms": 272.7118848601879
|
|
},
|
|
{
|
|
"e2e_ms": 21101.69867612793,
|
|
"request_id": 18,
|
|
"tpot_ms": 163.50974377121332,
|
|
"ttft_ms": 335.9612171838391
|
|
},
|
|
{
|
|
"e2e_ms": 20890.778369055995,
|
|
"request_id": 19,
|
|
"tpot_ms": 162.16628083276893,
|
|
"ttft_ms": 295.66070329433813
|
|
},
|
|
{
|
|
"e2e_ms": 20782.558056179907,
|
|
"request_id": 20,
|
|
"tpot_ms": 160.7985342727971,
|
|
"ttft_ms": 361.14420353467835
|
|
},
|
|
{
|
|
"e2e_ms": 20571.512469390585,
|
|
"request_id": 21,
|
|
"tpot_ms": 159.44598375979533,
|
|
"ttft_ms": 321.8725318965792
|
|
},
|
|
{
|
|
"e2e_ms": 20360.405278761264,
|
|
"request_id": 22,
|
|
"tpot_ms": 158.09565605925098,
|
|
"ttft_ms": 282.2569592363866
|
|
},
|
|
{
|
|
"e2e_ms": 20251.586130359967,
|
|
"request_id": 23,
|
|
"tpot_ms": 156.71227679254048,
|
|
"ttft_ms": 349.12697770732314
|
|
},
|
|
{
|
|
"e2e_ms": 20042.10448060375,
|
|
"request_id": 24,
|
|
"tpot_ms": 155.37142770924103,
|
|
"ttft_ms": 309.93316153013774
|
|
},
|
|
{
|
|
"e2e_ms": 19934.008109113514,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.05960281316982,
|
|
"ttft_ms": 368.43855184094565
|
|
},
|
|
{
|
|
"e2e_ms": 19722.984661067087,
|
|
"request_id": 26,
|
|
"tpot_ms": 152.73931080974432,
|
|
"ttft_ms": 325.0921882295579
|
|
},
|
|
{
|
|
"e2e_ms": 19511.768043076878,
|
|
"request_id": 27,
|
|
"tpot_ms": 151.41621155599998,
|
|
"ttft_ms": 281.9091754648788
|
|
},
|
|
{
|
|
"e2e_ms": 19403.203575383635,
|
|
"request_id": 28,
|
|
"tpot_ms": 150.09414530000853,
|
|
"ttft_ms": 341.2471222825513
|
|
},
|
|
{
|
|
"e2e_ms": 19191.714653347688,
|
|
"request_id": 29,
|
|
"tpot_ms": 148.76487274343572,
|
|
"ttft_ms": 298.57581493135046
|
|
},
|
|
{
|
|
"e2e_ms": 19082.708845359608,
|
|
"request_id": 30,
|
|
"tpot_ms": 147.43434256455518,
|
|
"ttft_ms": 358.5473396611025
|
|
},
|
|
{
|
|
"e2e_ms": 18871.078709760895,
|
|
"request_id": 31,
|
|
"tpot_ms": 146.10277278695904,
|
|
"ttft_ms": 316.0265658170971
|
|
},
|
|
{
|
|
"e2e_ms": 18659.370447496967,
|
|
"request_id": 32,
|
|
"tpot_ms": 144.7695321688903,
|
|
"ttft_ms": 273.6398620479008
|
|
},
|
|
{
|
|
"e2e_ms": 18549.637648096734,
|
|
"request_id": 33,
|
|
"tpot_ms": 143.43132538741386,
|
|
"ttft_ms": 333.85932389517234
|
|
},
|
|
{
|
|
"e2e_ms": 18337.808235788358,
|
|
"request_id": 34,
|
|
"tpot_ms": 142.0958670025312,
|
|
"ttft_ms": 291.63312646689656
|
|
},
|
|
{
|
|
"e2e_ms": 18227.80864738452,
|
|
"request_id": 35,
|
|
"tpot_ms": 140.75352098043697,
|
|
"ttft_ms": 352.1114828690255
|
|
},
|
|
{
|
|
"e2e_ms": 18015.40726100407,
|
|
"request_id": 36,
|
|
"tpot_ms": 139.41149279948888,
|
|
"ttft_ms": 310.14767546898446
|
|
},
|
|
{
|
|
"e2e_ms": 17904.857868774245,
|
|
"request_id": 37,
|
|
"tpot_ms": 138.06211837292886,
|
|
"ttft_ms": 370.9688354122758
|
|
},
|
|
{
|
|
"e2e_ms": 17692.008101355255,
|
|
"request_id": 38,
|
|
"tpot_ms": 136.71510258553306,
|
|
"ttft_ms": 329.1900729925583
|
|
},
|
|
{
|
|
"e2e_ms": 17479.148877024498,
|
|
"request_id": 39,
|
|
"tpot_ms": 135.36749298254742,
|
|
"ttft_ms": 287.47726824097344
|
|
},
|
|
{
|
|
"e2e_ms": 17381.961391787096,
|
|
"request_id": 40,
|
|
"tpot_ms": 134.1089349239123,
|
|
"ttft_ms": 350.1266564502323
|
|
},
|
|
{
|
|
"e2e_ms": 17178.01129231477,
|
|
"request_id": 41,
|
|
"tpot_ms": 132.8327622110564,
|
|
"ttft_ms": 308.2504915106092
|
|
},
|
|
{
|
|
"e2e_ms": 17080.086531895085,
|
|
"request_id": 42,
|
|
"tpot_ms": 131.5805170059729,
|
|
"ttft_ms": 369.36087213652866
|
|
},
|
|
{
|
|
"e2e_ms": 16876.654129344566,
|
|
"request_id": 43,
|
|
"tpot_ms": 130.30640250229436,
|
|
"ttft_ms": 327.74101155318294
|
|
},
|
|
{
|
|
"e2e_ms": 16671.422925287778,
|
|
"request_id": 44,
|
|
"tpot_ms": 129.0178430005707,
|
|
"ttft_ms": 286.15686421529676
|
|
},
|
|
{
|
|
"e2e_ms": 16571.761179006833,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.74972161297816,
|
|
"ttft_ms": 347.5465341586066
|
|
},
|
|
{
|
|
"e2e_ms": 16364.721381165033,
|
|
"request_id": 46,
|
|
"tpot_ms": 126.44572566423672,
|
|
"ttft_ms": 306.1142218069683
|
|
},
|
|
{
|
|
"e2e_ms": 16261.449604637399,
|
|
"request_id": 47,
|
|
"tpot_ms": 125.14700916940275,
|
|
"ttft_ms": 367.77944012325037
|
|
},
|
|
{
|
|
"e2e_ms": 16053.400400183471,
|
|
"request_id": 48,
|
|
"tpot_ms": 123.8340136000885,
|
|
"ttft_ms": 326.48067297223093
|
|
},
|
|
{
|
|
"e2e_ms": 15845.03658651726,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.50141376236263,
|
|
"ttft_ms": 287.35703869720555
|
|
},
|
|
{
|
|
"e2e_ms": 15739.724302020804,
|
|
"request_id": 50,
|
|
"tpot_ms": 121.15600683324593,
|
|
"ttft_ms": 352.9114341985711
|
|
},
|
|
{
|
|
"e2e_ms": 15530.891082399023,
|
|
"request_id": 51,
|
|
"tpot_ms": 119.81946138926746,
|
|
"ttft_ms": 313.8194859620569
|
|
},
|
|
{
|
|
"e2e_ms": 15319.199082162215,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.45427332697808,
|
|
"ttft_ms": 275.5063696359983
|
|
},
|
|
{
|
|
"e2e_ms": 15206.960333133673,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.04949498375757,
|
|
"ttft_ms": 341.6744701964625
|
|
},
|
|
{
|
|
"e2e_ms": 14987.9081257507,
|
|
"request_id": 54,
|
|
"tpot_ms": 115.62635253449054,
|
|
"ttft_ms": 303.36135387040383
|
|
},
|
|
{
|
|
"e2e_ms": 14861.077413909577,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.10642655651417,
|
|
"ttft_ms": 369.5612412322795
|
|
},
|
|
{
|
|
"e2e_ms": 14633.756981193535,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.61817997075843,
|
|
"ttft_ms": 331.2481249072157
|
|
},
|
|
{
|
|
"e2e_ms": 14403.991359276923,
|
|
"request_id": 57,
|
|
"tpot_ms": 111.11042963695559,
|
|
"ttft_ms": 292.9667953835633
|
|
},
|
|
{
|
|
"e2e_ms": 14264.847964808048,
|
|
"request_id": 58,
|
|
"tpot_ms": 109.49330311228506,
|
|
"ttft_ms": 359.1984695478452
|
|
},
|
|
{
|
|
"e2e_ms": 14029.729570522895,
|
|
"request_id": 59,
|
|
"tpot_ms": 107.94340496454889,
|
|
"ttft_ms": 320.91714002518756
|
|
},
|
|
{
|
|
"e2e_ms": 13793.064611419886,
|
|
"request_id": 60,
|
|
"tpot_ms": 106.38108660628622,
|
|
"ttft_ms": 282.66661242153646
|
|
},
|
|
{
|
|
"e2e_ms": 13648.251078198633,
|
|
"request_id": 61,
|
|
"tpot_ms": 104.71931331978594,
|
|
"ttft_ms": 348.89828658581837
|
|
},
|
|
{
|
|
"e2e_ms": 13412.62809971594,
|
|
"request_id": 62,
|
|
"tpot_ms": 103.16519953332896,
|
|
"ttft_ms": 310.647758983162
|
|
},
|
|
{
|
|
"e2e_ms": 13260.053320226874,
|
|
"request_id": 63,
|
|
"tpot_ms": 101.44207153668842,
|
|
"ttft_ms": 376.9102350674451
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.203125,
|
|
"passed": 13
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.546875,
|
|
"passed": 35
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "0b9f30d883de83b4af69dd114bd5d2bb974b7fd1f01d45f913bf5049e4b72a86"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 41.42983865737915,
|
|
"offered_request_rate": 3.2,
|
|
"request_metrics_sha256": "bfba1ae71cfb6d0595614b14bc49ac47591bfad6792be0a3d98d2c98b16012e8",
|
|
"request_rate_per_gpu": 0.8,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 24003.81066522886,
|
|
"request_id": 0,
|
|
"tpot_ms": 186.89946612460176,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 23901.058156177503,
|
|
"request_id": 1,
|
|
"tpot_ms": 185.74139588389912,
|
|
"ttft_ms": 311.9008789223171
|
|
},
|
|
{
|
|
"e2e_ms": 23798.190109844723,
|
|
"request_id": 2,
|
|
"tpot_ms": 184.56225305811301,
|
|
"ttft_ms": 358.7839714643697
|
|
},
|
|
{
|
|
"e2e_ms": 23590.44639829378,
|
|
"request_id": 3,
|
|
"tpot_ms": 183.2673714648551,
|
|
"ttft_ms": 315.49022225718517
|
|
},
|
|
{
|
|
"e2e_ms": 23382.70131446869,
|
|
"request_id": 4,
|
|
"tpot_ms": 181.9732705315891,
|
|
"ttft_ms": 272.09595695687216
|
|
},
|
|
{
|
|
"e2e_ms": 23279.64954297851,
|
|
"request_id": 5,
|
|
"tpot_ms": 180.78914255292864,
|
|
"ttft_ms": 319.4284387565705
|
|
},
|
|
{
|
|
"e2e_ms": 23071.973865900134,
|
|
"request_id": 6,
|
|
"tpot_ms": 179.49398625037213,
|
|
"ttft_ms": 276.2376121028707
|
|
},
|
|
{
|
|
"e2e_ms": 22969.022726489202,
|
|
"request_id": 7,
|
|
"tpot_ms": 178.2861180199918,
|
|
"ttft_ms": 326.6857379502426
|
|
},
|
|
{
|
|
"e2e_ms": 22761.29942646063,
|
|
"request_id": 8,
|
|
"tpot_ms": 176.98924979786668,
|
|
"ttft_ms": 283.66470213156344
|
|
},
|
|
{
|
|
"e2e_ms": 22658.264587165988,
|
|
"request_id": 9,
|
|
"tpot_ms": 175.7551174352214,
|
|
"ttft_ms": 337.36467289287384
|
|
},
|
|
{
|
|
"e2e_ms": 22450.304224673095,
|
|
"request_id": 10,
|
|
"tpot_ms": 174.4550575162989,
|
|
"ttft_ms": 294.51192010313275
|
|
},
|
|
{
|
|
"e2e_ms": 22346.821408551063,
|
|
"request_id": 11,
|
|
"tpot_ms": 173.1774539652428,
|
|
"ttft_ms": 353.28475496522805
|
|
},
|
|
{
|
|
"e2e_ms": 22138.79812545183,
|
|
"request_id": 12,
|
|
"tpot_ms": 171.87427427697472,
|
|
"ttft_ms": 310.7652922760442
|
|
},
|
|
{
|
|
"e2e_ms": 22035.248814705075,
|
|
"request_id": 13,
|
|
"tpot_ms": 170.57508935568035,
|
|
"ttft_ms": 372.21246653366967
|
|
},
|
|
{
|
|
"e2e_ms": 21827.324416126074,
|
|
"request_id": 14,
|
|
"tpot_ms": 169.27188182433355,
|
|
"ttft_ms": 329.79542443571484
|
|
},
|
|
{
|
|
"e2e_ms": 21619.373671552865,
|
|
"request_id": 15,
|
|
"tpot_ms": 167.96821838814947,
|
|
"ttft_ms": 287.40993625788303
|
|
},
|
|
{
|
|
"e2e_ms": 21512.77581709026,
|
|
"request_id": 16,
|
|
"tpot_ms": 166.64216905152279,
|
|
"ttft_ms": 349.2203475468658
|
|
},
|
|
{
|
|
"e2e_ms": 21303.255969153783,
|
|
"request_id": 17,
|
|
"tpot_ms": 165.32482604305434,
|
|
"ttft_ms": 307.003061685883
|
|
},
|
|
{
|
|
"e2e_ms": 21196.567333974876,
|
|
"request_id": 18,
|
|
"tpot_ms": 163.99164288866731,
|
|
"ttft_ms": 369.62868711412875
|
|
},
|
|
{
|
|
"e2e_ms": 20986.94051377241,
|
|
"request_id": 19,
|
|
"tpot_ms": 162.67002407488405,
|
|
"ttft_ms": 327.84745626213316
|
|
},
|
|
{
|
|
"e2e_ms": 20777.255965035714,
|
|
"request_id": 20,
|
|
"tpot_ms": 161.34765321373814,
|
|
"ttft_ms": 286.10400689097213
|
|
},
|
|
{
|
|
"e2e_ms": 20670.377044913395,
|
|
"request_id": 21,
|
|
"tpot_ms": 159.9951027007364,
|
|
"ttft_ms": 350.99900191987297
|
|
},
|
|
{
|
|
"e2e_ms": 20460.656782932234,
|
|
"request_id": 22,
|
|
"tpot_ms": 158.67196467076286,
|
|
"ttft_ms": 309.3172697453532
|
|
},
|
|
{
|
|
"e2e_ms": 20353.38846177537,
|
|
"request_id": 23,
|
|
"tpot_ms": 157.31793948495124,
|
|
"ttft_ms": 374.01014718656
|
|
},
|
|
{
|
|
"e2e_ms": 20144.48097023076,
|
|
"request_id": 24,
|
|
"tpot_ms": 155.99981541047092,
|
|
"ttft_ms": 332.5044131009545
|
|
},
|
|
{
|
|
"e2e_ms": 19934.83773528968,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.6729368346084,
|
|
"ttft_ms": 291.37475729441144
|
|
},
|
|
{
|
|
"e2e_ms": 19827.980953909257,
|
|
"request_id": 26,
|
|
"tpot_ms": 153.35264483118294,
|
|
"ttft_ms": 352.1950603490236
|
|
},
|
|
{
|
|
"e2e_ms": 19618.20597824765,
|
|
"request_id": 27,
|
|
"tpot_ms": 152.02412197401114,
|
|
"ttft_ms": 311.1424875482367
|
|
},
|
|
{
|
|
"e2e_ms": 19511.107444230154,
|
|
"request_id": 28,
|
|
"tpot_ms": 150.6975599886974,
|
|
"ttft_ms": 372.5173256655836
|
|
},
|
|
{
|
|
"e2e_ms": 19301.196316545185,
|
|
"request_id": 29,
|
|
"tpot_ms": 149.36537894653213,
|
|
"ttft_ms": 331.7931903356044
|
|
},
|
|
{
|
|
"e2e_ms": 19091.249936104825,
|
|
"request_id": 30,
|
|
"tpot_ms": 148.03267778881258,
|
|
"ttft_ms": 291.0998569256247
|
|
},
|
|
{
|
|
"e2e_ms": 18983.78646717311,
|
|
"request_id": 31,
|
|
"tpot_ms": 146.7011080112165,
|
|
"ttft_ms": 352.7457497486193
|
|
},
|
|
{
|
|
"e2e_ms": 18773.765669374145,
|
|
"request_id": 32,
|
|
"tpot_ms": 145.36784483585674,
|
|
"ttft_ms": 312.0493752203348
|
|
},
|
|
{
|
|
"e2e_ms": 18665.888247551986,
|
|
"request_id": 33,
|
|
"tpot_ms": 144.03124523287858,
|
|
"ttft_ms": 373.9201029764097
|
|
},
|
|
{
|
|
"e2e_ms": 18455.8068747313,
|
|
"request_id": 34,
|
|
"tpot_ms": 142.6968275070543,
|
|
"ttft_ms": 333.30978133540157
|
|
},
|
|
{
|
|
"e2e_ms": 18245.529234152025,
|
|
"request_id": 35,
|
|
"tpot_ms": 141.3602386069844,
|
|
"ttft_ms": 292.7789310650048
|
|
},
|
|
{
|
|
"e2e_ms": 18137.294514438578,
|
|
"request_id": 36,
|
|
"tpot_ms": 140.01821042603632,
|
|
"ttft_ms": 354.98179033196385
|
|
},
|
|
{
|
|
"e2e_ms": 17926.83360591763,
|
|
"request_id": 37,
|
|
"tpot_ms": 138.67967407469433,
|
|
"ttft_ms": 314.5149984314486
|
|
},
|
|
{
|
|
"e2e_ms": 17716.242055541137,
|
|
"request_id": 38,
|
|
"tpot_ms": 137.3398665125213,
|
|
"ttft_ms": 274.0790084509328
|
|
},
|
|
{
|
|
"e2e_ms": 17607.549497877386,
|
|
"request_id": 39,
|
|
"tpot_ms": 135.99225690953574,
|
|
"ttft_ms": 336.53287036634794
|
|
},
|
|
{
|
|
"e2e_ms": 17401.54255835392,
|
|
"request_id": 40,
|
|
"tpot_ms": 134.6880568492608,
|
|
"ttft_ms": 296.1593384978016
|
|
},
|
|
{
|
|
"e2e_ms": 17301.7591255476,
|
|
"request_id": 41,
|
|
"tpot_ms": 133.41188413640484,
|
|
"ttft_ms": 358.4498402241838
|
|
},
|
|
{
|
|
"e2e_ms": 17095.506427630044,
|
|
"request_id": 42,
|
|
"tpot_ms": 132.10522228512968,
|
|
"ttft_ms": 318.1431974185731
|
|
},
|
|
{
|
|
"e2e_ms": 16889.623559688287,
|
|
"request_id": 43,
|
|
"tpot_ms": 130.8012299461049,
|
|
"ttft_ms": 277.86735653296194
|
|
},
|
|
{
|
|
"e2e_ms": 16788.559022297493,
|
|
"request_id": 44,
|
|
"tpot_ms": 129.5126704443812,
|
|
"ttft_ms": 340.449875861081
|
|
},
|
|
{
|
|
"e2e_ms": 16581.727329092846,
|
|
"request_id": 45,
|
|
"tpot_ms": 128.200721970688,
|
|
"ttft_ms": 300.23563881547057
|
|
},
|
|
{
|
|
"e2e_ms": 16478.854197918037,
|
|
"request_id": 46,
|
|
"tpot_ms": 126.8967260219465,
|
|
"ttft_ms": 362.9699931308306
|
|
},
|
|
{
|
|
"e2e_ms": 16270.819161297495,
|
|
"request_id": 47,
|
|
"tpot_ms": 125.57481733363996,
|
|
"ttft_ms": 322.8173599252209
|
|
},
|
|
{
|
|
"e2e_ms": 16062.627892404027,
|
|
"request_id": 48,
|
|
"tpot_ms": 124.25122687214136,
|
|
"ttft_ms": 282.72207964207576
|
|
},
|
|
{
|
|
"e2e_ms": 15958.430745404816,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.91862703441548,
|
|
"ttft_ms": 347.7651120340504
|
|
},
|
|
{
|
|
"e2e_ms": 15749.715539461335,
|
|
"request_id": 50,
|
|
"tpot_ms": 121.58574387256766,
|
|
"ttft_ms": 308.3260676452415
|
|
},
|
|
{
|
|
"e2e_ms": 15645.048986506545,
|
|
"request_id": 51,
|
|
"tpot_ms": 120.24919842858912,
|
|
"ttft_ms": 373.40078607572735
|
|
},
|
|
{
|
|
"e2e_ms": 15435.036319721643,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.8997102377205,
|
|
"ttft_ms": 334.77311953113895
|
|
},
|
|
{
|
|
"e2e_ms": 15224.012292267464,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.54201604221191,
|
|
"ttft_ms": 296.1762549065519
|
|
},
|
|
{
|
|
"e2e_ms": 15109.12675155149,
|
|
"request_id": 54,
|
|
"tpot_ms": 116.11789378026353,
|
|
"ttft_ms": 362.1542414580219
|
|
},
|
|
{
|
|
"e2e_ms": 14893.238736493451,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.72141540015762,
|
|
"ttft_ms": 323.6189806734337
|
|
},
|
|
{
|
|
"e2e_ms": 14675.411853468433,
|
|
"request_id": 56,
|
|
"tpot_ms": 113.30942780834319,
|
|
"ttft_ms": 285.1145218088469
|
|
},
|
|
{
|
|
"e2e_ms": 14549.812898218817,
|
|
"request_id": 57,
|
|
"tpot_ms": 111.79839094375423,
|
|
"ttft_ms": 351.4172483620292
|
|
},
|
|
{
|
|
"e2e_ms": 14329.820655618192,
|
|
"request_id": 58,
|
|
"tpot_ms": 110.36886820693505,
|
|
"ttft_ms": 312.97439333744137
|
|
},
|
|
{
|
|
"e2e_ms": 14108.094791808619,
|
|
"request_id": 59,
|
|
"tpot_ms": 108.92545237461232,
|
|
"ttft_ms": 274.56234023285475
|
|
},
|
|
{
|
|
"e2e_ms": 13975.596499372607,
|
|
"request_id": 60,
|
|
"tpot_ms": 107.35951607044248,
|
|
"ttft_ms": 340.93795842641225
|
|
},
|
|
{
|
|
"e2e_ms": 13753.714210521188,
|
|
"request_id": 61,
|
|
"tpot_ms": 105.91438347527057,
|
|
"ttft_ms": 302.58750916182464
|
|
},
|
|
{
|
|
"e2e_ms": 13622.257898704496,
|
|
"request_id": 62,
|
|
"tpot_ms": 104.35568588071192,
|
|
"ttft_ms": 369.08579185408286
|
|
},
|
|
{
|
|
"e2e_ms": 13397.788527763809,
|
|
"request_id": 63,
|
|
"tpot_ms": 102.88969749082138,
|
|
"ttft_ms": 330.7969464294942
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.1875,
|
|
"passed": 12
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.546875,
|
|
"passed": 35
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.90625,
|
|
"passed": 58
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "96c4e04e8f133e66449a5567fcde568f27af44bd9f5d1cadeade66b771a8f0f3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"loads": [
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 34.255025148391724,
|
|
"offered_request_rate": 0.1,
|
|
"request_metrics_sha256": "b45cd3a473a98bee1127b2315d0c0c59ded580275f6e6d10a3e5a52d85d24de4",
|
|
"request_rate_per_gpu": 0.025,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 11656.879780530655,
|
|
"request_id": 0,
|
|
"tpot_ms": 89.67953789863164,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 11729.945143691864,
|
|
"request_id": 1,
|
|
"tpot_ms": 89.99959080602144,
|
|
"ttft_ms": 299.9971113271407
|
|
},
|
|
{
|
|
"e2e_ms": 11712.730146443742,
|
|
"request_id": 2,
|
|
"tpot_ms": 90.00056214193414,
|
|
"ttft_ms": 282.65875441810806
|
|
},
|
|
{
|
|
"e2e_ms": 11785.795509604945,
|
|
"request_id": 3,
|
|
"tpot_ms": 89.99959080602139,
|
|
"ttft_ms": 355.8474772402285
|
|
},
|
|
{
|
|
"e2e_ms": 11770.95360336775,
|
|
"request_id": 4,
|
|
"tpot_ms": 90.01924789792568,
|
|
"ttft_ms": 338.50912033118874
|
|
},
|
|
{
|
|
"e2e_ms": 11756.111697130556,
|
|
"request_id": 5,
|
|
"tpot_ms": 90.0192478979257,
|
|
"ttft_ms": 323.66721409399446
|
|
},
|
|
{
|
|
"e2e_ms": 11741.269790893028,
|
|
"request_id": 6,
|
|
"tpot_ms": 90.01924789792305,
|
|
"ttft_ms": 308.8253078568002
|
|
},
|
|
{
|
|
"e2e_ms": 11726.427884655508,
|
|
"request_id": 7,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 293.98340161927194
|
|
},
|
|
{
|
|
"e2e_ms": 11709.212887407048,
|
|
"request_id": 8,
|
|
"tpot_ms": 90.00056214193147,
|
|
"ttft_ms": 279.1414953817508
|
|
},
|
|
{
|
|
"e2e_ms": 11782.278250567928,
|
|
"request_id": 9,
|
|
"tpot_ms": 89.99959080601886,
|
|
"ttft_ms": 352.3302182035337
|
|
},
|
|
{
|
|
"e2e_ms": 11767.436344330406,
|
|
"request_id": 10,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 334.9918612941707
|
|
},
|
|
{
|
|
"e2e_ms": 11752.594438092885,
|
|
"request_id": 11,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 320.14995505664956
|
|
},
|
|
{
|
|
"e2e_ms": 11737.752531855705,
|
|
"request_id": 12,
|
|
"tpot_ms": 90.01924789792581,
|
|
"ttft_ms": 305.30804881912843
|
|
},
|
|
{
|
|
"e2e_ms": 11722.91062561817,
|
|
"request_id": 13,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 290.46614258194836
|
|
},
|
|
{
|
|
"e2e_ms": 11705.69562836971,
|
|
"request_id": 14,
|
|
"tpot_ms": 90.00056214193148,
|
|
"ttft_ms": 275.624236344413
|
|
},
|
|
{
|
|
"e2e_ms": 11778.76099153056,
|
|
"request_id": 15,
|
|
"tpot_ms": 89.99959080601862,
|
|
"ttft_ms": 348.8129591661959
|
|
},
|
|
{
|
|
"e2e_ms": 11763.919085293026,
|
|
"request_id": 16,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 331.4746022568045
|
|
},
|
|
{
|
|
"e2e_ms": 11749.07717905549,
|
|
"request_id": 17,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 316.63269601926913
|
|
},
|
|
{
|
|
"e2e_ms": 11734.235272817954,
|
|
"request_id": 18,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 301.7907897817338
|
|
},
|
|
{
|
|
"e2e_ms": 11719.39336658042,
|
|
"request_id": 19,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 286.94888354419845
|
|
},
|
|
{
|
|
"e2e_ms": 11702.178369331961,
|
|
"request_id": 20,
|
|
"tpot_ms": 90.00056214193148,
|
|
"ttft_ms": 272.1069773066631
|
|
},
|
|
{
|
|
"e2e_ms": 11775.24373249281,
|
|
"request_id": 21,
|
|
"tpot_ms": 89.99959080601862,
|
|
"ttft_ms": 345.295700128446
|
|
},
|
|
{
|
|
"e2e_ms": 11760.401826255276,
|
|
"request_id": 22,
|
|
"tpot_ms": 90.01924789792301,
|
|
"ttft_ms": 327.95734321905456
|
|
},
|
|
{
|
|
"e2e_ms": 11745.55992001774,
|
|
"request_id": 23,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 313.1154369815192
|
|
},
|
|
{
|
|
"e2e_ms": 11730.718013780204,
|
|
"request_id": 24,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 298.2735307439839
|
|
},
|
|
{
|
|
"e2e_ms": 11713.50301653041,
|
|
"request_id": 25,
|
|
"tpot_ms": 90.00056214192095,
|
|
"ttft_ms": 283.43162450644854
|
|
},
|
|
{
|
|
"e2e_ms": 11786.568379689925,
|
|
"request_id": 26,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 356.6203473268956
|
|
},
|
|
{
|
|
"e2e_ms": 11771.726473451054,
|
|
"request_id": 27,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 339.28199041616836
|
|
},
|
|
{
|
|
"e2e_ms": 11756.884567212182,
|
|
"request_id": 28,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 324.4400841772972
|
|
},
|
|
{
|
|
"e2e_ms": 11742.042660973311,
|
|
"request_id": 29,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 309.59817793842603
|
|
},
|
|
{
|
|
"e2e_ms": 11727.20075473444,
|
|
"request_id": 30,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 294.7562716995549
|
|
},
|
|
{
|
|
"e2e_ms": 11709.985757484617,
|
|
"request_id": 31,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 279.9143654606837
|
|
},
|
|
{
|
|
"e2e_ms": 11783.051120644132,
|
|
"request_id": 32,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 353.1030882811024
|
|
},
|
|
{
|
|
"e2e_ms": 11768.20921440526,
|
|
"request_id": 33,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 335.7647313703751
|
|
},
|
|
{
|
|
"e2e_ms": 11753.36730816639,
|
|
"request_id": 34,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 320.92282513150394
|
|
},
|
|
{
|
|
"e2e_ms": 11738.525401927518,
|
|
"request_id": 35,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 306.0809188926328
|
|
},
|
|
{
|
|
"e2e_ms": 11723.683495688647,
|
|
"request_id": 36,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 291.2390126537616
|
|
},
|
|
{
|
|
"e2e_ms": 11706.468498438824,
|
|
"request_id": 37,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 276.39710641489046
|
|
},
|
|
{
|
|
"e2e_ms": 11779.533861598338,
|
|
"request_id": 38,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 349.5858292353091
|
|
},
|
|
{
|
|
"e2e_ms": 11764.691955359467,
|
|
"request_id": 39,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 332.24747232458185
|
|
},
|
|
{
|
|
"e2e_ms": 11749.850049120596,
|
|
"request_id": 40,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 317.4055660857107
|
|
},
|
|
{
|
|
"e2e_ms": 11735.008142881725,
|
|
"request_id": 41,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 302.5636598468395
|
|
},
|
|
{
|
|
"e2e_ms": 11720.166236642854,
|
|
"request_id": 42,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 287.72175360796837
|
|
},
|
|
{
|
|
"e2e_ms": 11702.95123939303,
|
|
"request_id": 43,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 272.8798473690972
|
|
},
|
|
{
|
|
"e2e_ms": 11776.016602552545,
|
|
"request_id": 44,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 346.06857018951587
|
|
},
|
|
{
|
|
"e2e_ms": 11761.174696313674,
|
|
"request_id": 45,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 328.7302132787886
|
|
},
|
|
{
|
|
"e2e_ms": 11746.332790074803,
|
|
"request_id": 46,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 313.88830703991744
|
|
},
|
|
{
|
|
"e2e_ms": 11731.490883835932,
|
|
"request_id": 47,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 299.0464008010463
|
|
},
|
|
{
|
|
"e2e_ms": 11716.64897759706,
|
|
"request_id": 48,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 284.2044945621751
|
|
},
|
|
{
|
|
"e2e_ms": 11699.433980347238,
|
|
"request_id": 49,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 269.36258832330395
|
|
},
|
|
{
|
|
"e2e_ms": 11772.499343506752,
|
|
"request_id": 50,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 342.5513111437226
|
|
},
|
|
{
|
|
"e2e_ms": 11757.657437270042,
|
|
"request_id": 51,
|
|
"tpot_ms": 90.0192478979295,
|
|
"ttft_ms": 325.21295423299534
|
|
},
|
|
{
|
|
"e2e_ms": 11742.815531033444,
|
|
"request_id": 52,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 310.3710479967958
|
|
},
|
|
{
|
|
"e2e_ms": 11727.973624796847,
|
|
"request_id": 53,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 295.5291417601984
|
|
},
|
|
{
|
|
"e2e_ms": 11710.758627549352,
|
|
"request_id": 54,
|
|
"tpot_ms": 90.00056214193505,
|
|
"ttft_ms": 280.687235523601
|
|
},
|
|
{
|
|
"e2e_ms": 11783.823990711198,
|
|
"request_id": 55,
|
|
"tpot_ms": 89.99959080602287,
|
|
"ttft_ms": 353.8759583462934
|
|
},
|
|
{
|
|
"e2e_ms": 11768.982084474601,
|
|
"request_id": 56,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 336.5376014379535
|
|
},
|
|
{
|
|
"e2e_ms": 11754.140178238004,
|
|
"request_id": 57,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 321.6956952013561
|
|
},
|
|
{
|
|
"e2e_ms": 11739.298272001406,
|
|
"request_id": 58,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 306.8537889647587
|
|
},
|
|
{
|
|
"e2e_ms": 11724.456365764809,
|
|
"request_id": 59,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 292.01188272816125
|
|
},
|
|
{
|
|
"e2e_ms": 11707.241368517316,
|
|
"request_id": 60,
|
|
"tpot_ms": 90.00056214193505,
|
|
"ttft_ms": 277.16997649156383
|
|
},
|
|
{
|
|
"e2e_ms": 11780.30673167916,
|
|
"request_id": 61,
|
|
"tpot_ms": 89.99959080602287,
|
|
"ttft_ms": 350.3586993142562
|
|
},
|
|
{
|
|
"e2e_ms": 11765.464825442563,
|
|
"request_id": 62,
|
|
"tpot_ms": 90.01924789792635,
|
|
"ttft_ms": 333.0203424059164
|
|
},
|
|
{
|
|
"e2e_ms": 11530.943387303068,
|
|
"request_id": 63,
|
|
"tpot_ms": 88.28948780420275,
|
|
"ttft_ms": 318.17843616931896
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "37e166453683700c9df7abffa278431119205a29c83e92211ffa69d90e1dca0e"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 26.24324607849121,
|
|
"offered_request_rate": 0.2,
|
|
"request_metrics_sha256": "7930ffb53e440b9eefa724912c777842a9c33cefdb46d9014e1c5003223221d2",
|
|
"request_rate_per_gpu": 0.05,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 11975.67900227787,
|
|
"request_id": 0,
|
|
"tpot_ms": 92.18976799112939,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 12131.199338988026,
|
|
"request_id": 1,
|
|
"tpot_ms": 93.2890801728389,
|
|
"ttft_ms": 283.4861570374869
|
|
},
|
|
{
|
|
"e2e_ms": 12196.411822684866,
|
|
"request_id": 2,
|
|
"tpot_ms": 93.31228634935276,
|
|
"ttft_ms": 345.75145631706584
|
|
},
|
|
{
|
|
"e2e_ms": 12171.220586311463,
|
|
"request_id": 3,
|
|
"tpot_ms": 93.31547315047263,
|
|
"ttft_ms": 320.15549620144054
|
|
},
|
|
{
|
|
"e2e_ms": 12145.933482881133,
|
|
"request_id": 4,
|
|
"tpot_ms": 93.31640326087744,
|
|
"ttft_ms": 294.75026874969853
|
|
},
|
|
{
|
|
"e2e_ms": 12120.646379450847,
|
|
"request_id": 5,
|
|
"tpot_ms": 93.31564840216181,
|
|
"ttft_ms": 269.5590323762964
|
|
},
|
|
{
|
|
"e2e_ms": 12185.858863147729,
|
|
"request_id": 6,
|
|
"tpot_ms": 93.31471829175732,
|
|
"ttft_ms": 334.88964009454844
|
|
},
|
|
{
|
|
"e2e_ms": 12160.571759717443,
|
|
"request_id": 7,
|
|
"tpot_ms": 93.31471829175764,
|
|
"ttft_ms": 309.6025366642223
|
|
},
|
|
{
|
|
"e2e_ms": 12135.284656287155,
|
|
"request_id": 8,
|
|
"tpot_ms": 93.31564840216245,
|
|
"ttft_ms": 284.19730921252295
|
|
},
|
|
{
|
|
"e2e_ms": 12200.497139984038,
|
|
"request_id": 9,
|
|
"tpot_ms": 93.31471829175764,
|
|
"ttft_ms": 349.52791693081764
|
|
},
|
|
{
|
|
"e2e_ms": 12175.305903610677,
|
|
"request_id": 10,
|
|
"tpot_ms": 93.3154731504736,
|
|
"ttft_ms": 324.2408135005306
|
|
},
|
|
{
|
|
"e2e_ms": 12150.018800180305,
|
|
"request_id": 11,
|
|
"tpot_ms": 93.31640326087775,
|
|
"ttft_ms": 298.8355860488312
|
|
},
|
|
{
|
|
"e2e_ms": 12124.731696749934,
|
|
"request_id": 12,
|
|
"tpot_ms": 93.31564840216113,
|
|
"ttft_ms": 273.6443496754717
|
|
},
|
|
{
|
|
"e2e_ms": 12189.94418044673,
|
|
"request_id": 13,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 338.97495739375927
|
|
},
|
|
{
|
|
"e2e_ms": 12164.657077016358,
|
|
"request_id": 14,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 313.68785396338694
|
|
},
|
|
{
|
|
"e2e_ms": 12139.369973585985,
|
|
"request_id": 15,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 288.2826265116023
|
|
},
|
|
{
|
|
"e2e_ms": 12204.582457282782,
|
|
"request_id": 16,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 353.6132342298117
|
|
},
|
|
{
|
|
"e2e_ms": 12179.391220909338,
|
|
"request_id": 17,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 328.3261307994394
|
|
},
|
|
{
|
|
"e2e_ms": 12154.104117478966,
|
|
"request_id": 18,
|
|
"tpot_ms": 93.31640326087647,
|
|
"ttft_ms": 302.92090334765476
|
|
},
|
|
{
|
|
"e2e_ms": 12128.817014048593,
|
|
"request_id": 19,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 277.72966697421
|
|
},
|
|
{
|
|
"e2e_ms": 12194.02949774539,
|
|
"request_id": 20,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 343.0602746924194
|
|
},
|
|
{
|
|
"e2e_ms": 12168.742394315017,
|
|
"request_id": 21,
|
|
"tpot_ms": 93.31471829175567,
|
|
"ttft_ms": 317.7731712620471
|
|
},
|
|
{
|
|
"e2e_ms": 12143.455290884645,
|
|
"request_id": 22,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 292.36794381026243
|
|
},
|
|
{
|
|
"e2e_ms": 12208.667774581443,
|
|
"request_id": 23,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 357.69855152847185
|
|
},
|
|
{
|
|
"e2e_ms": 12183.476538207999,
|
|
"request_id": 24,
|
|
"tpot_ms": 93.31547315047165,
|
|
"ttft_ms": 332.4114480980995
|
|
},
|
|
{
|
|
"e2e_ms": 12158.189434777625,
|
|
"request_id": 25,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 307.0062206463149
|
|
},
|
|
{
|
|
"e2e_ms": 12132.902331347253,
|
|
"request_id": 26,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 281.8149842728701
|
|
},
|
|
{
|
|
"e2e_ms": 12198.11481504405,
|
|
"request_id": 27,
|
|
"tpot_ms": 93.31471829175567,
|
|
"ttft_ms": 347.1455919910795
|
|
},
|
|
{
|
|
"e2e_ms": 12172.923578670605,
|
|
"request_id": 28,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 321.8584885607072
|
|
},
|
|
{
|
|
"e2e_ms": 12147.636475240233,
|
|
"request_id": 29,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 296.45326110892256
|
|
},
|
|
{
|
|
"e2e_ms": 12122.349371809862,
|
|
"request_id": 30,
|
|
"tpot_ms": 93.3156484021605,
|
|
"ttft_ms": 271.2620247354778
|
|
},
|
|
{
|
|
"e2e_ms": 12187.561855506658,
|
|
"request_id": 31,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 336.5926324536872
|
|
},
|
|
{
|
|
"e2e_ms": 12162.274752076286,
|
|
"request_id": 32,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 311.3055290233149
|
|
},
|
|
{
|
|
"e2e_ms": 12136.987648645914,
|
|
"request_id": 33,
|
|
"tpot_ms": 93.3156484021605,
|
|
"ttft_ms": 285.90030157153024
|
|
},
|
|
{
|
|
"e2e_ms": 12202.20013234271,
|
|
"request_id": 34,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 351.23090928973966
|
|
},
|
|
{
|
|
"e2e_ms": 12177.008895969266,
|
|
"request_id": 35,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 325.94380585936733
|
|
},
|
|
{
|
|
"e2e_ms": 12151.721792538894,
|
|
"request_id": 36,
|
|
"tpot_ms": 93.31640326087647,
|
|
"ttft_ms": 300.5385784075827
|
|
},
|
|
{
|
|
"e2e_ms": 12126.43468910852,
|
|
"request_id": 37,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 275.3473420341379
|
|
},
|
|
{
|
|
"e2e_ms": 12191.647172805318,
|
|
"request_id": 38,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 340.67794975234733
|
|
},
|
|
{
|
|
"e2e_ms": 12166.360069374947,
|
|
"request_id": 39,
|
|
"tpot_ms": 93.3147182917557,
|
|
"ttft_ms": 315.390846321975
|
|
},
|
|
{
|
|
"e2e_ms": 12141.072965944573,
|
|
"request_id": 40,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 289.98561887019036
|
|
},
|
|
{
|
|
"e2e_ms": 12206.285449641371,
|
|
"request_id": 41,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 355.3162265883998
|
|
},
|
|
{
|
|
"e2e_ms": 12181.094213267927,
|
|
"request_id": 42,
|
|
"tpot_ms": 93.31547315047165,
|
|
"ttft_ms": 330.02912315802746
|
|
},
|
|
{
|
|
"e2e_ms": 12155.807109837553,
|
|
"request_id": 43,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 304.6238957062428
|
|
},
|
|
{
|
|
"e2e_ms": 12130.520006407181,
|
|
"request_id": 44,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 279.43265933279804
|
|
},
|
|
{
|
|
"e2e_ms": 12195.73249010398,
|
|
"request_id": 45,
|
|
"tpot_ms": 93.3147182917557,
|
|
"ttft_ms": 344.76326705100746
|
|
},
|
|
{
|
|
"e2e_ms": 12170.445386673606,
|
|
"request_id": 46,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 319.47616362063513
|
|
},
|
|
{
|
|
"e2e_ms": 12145.158283243234,
|
|
"request_id": 47,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 294.0709361688505
|
|
},
|
|
{
|
|
"e2e_ms": 12210.37076694003,
|
|
"request_id": 48,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 359.4015438870599
|
|
},
|
|
{
|
|
"e2e_ms": 12185.179530566586,
|
|
"request_id": 49,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 334.1144404566876
|
|
},
|
|
{
|
|
"e2e_ms": 12159.892427136241,
|
|
"request_id": 50,
|
|
"tpot_ms": 93.31640326087668,
|
|
"ttft_ms": 308.70921300490295
|
|
},
|
|
{
|
|
"e2e_ms": 12134.605323705899,
|
|
"request_id": 51,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 283.51797663145817
|
|
},
|
|
{
|
|
"e2e_ms": 12199.817807402724,
|
|
"request_id": 52,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 348.848584349696
|
|
},
|
|
{
|
|
"e2e_ms": 12174.626571029308,
|
|
"request_id": 53,
|
|
"tpot_ms": 93.3154731504721,
|
|
"ttft_ms": 323.5614809193521
|
|
},
|
|
{
|
|
"e2e_ms": 12149.339467598964,
|
|
"request_id": 54,
|
|
"tpot_ms": 93.31640326087691,
|
|
"ttft_ms": 298.1562534675959
|
|
},
|
|
{
|
|
"e2e_ms": 12124.05236416862,
|
|
"request_id": 55,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 272.96501709417953
|
|
},
|
|
{
|
|
"e2e_ms": 12189.264847865445,
|
|
"request_id": 56,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 338.2956248124174
|
|
},
|
|
{
|
|
"e2e_ms": 12163.977744435102,
|
|
"request_id": 57,
|
|
"tpot_ms": 93.31471829175614,
|
|
"ttft_ms": 313.00852138207347
|
|
},
|
|
{
|
|
"e2e_ms": 12138.690641004758,
|
|
"request_id": 58,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 287.60329393031725
|
|
},
|
|
{
|
|
"e2e_ms": 12203.903124701583,
|
|
"request_id": 59,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 352.9339016485551
|
|
},
|
|
{
|
|
"e2e_ms": 12178.711888328166,
|
|
"request_id": 60,
|
|
"tpot_ms": 93.31547315047209,
|
|
"ttft_ms": 327.6467982182112
|
|
},
|
|
{
|
|
"e2e_ms": 12153.424784897823,
|
|
"request_id": 61,
|
|
"tpot_ms": 93.31640326087691,
|
|
"ttft_ms": 302.24157076645497
|
|
},
|
|
{
|
|
"e2e_ms": 11949.074413391485,
|
|
"request_id": 62,
|
|
"tpot_ms": 91.90570140943657,
|
|
"ttft_ms": 277.0503343930386
|
|
},
|
|
{
|
|
"e2e_ms": 11702.728382593137,
|
|
"request_id": 63,
|
|
"tpot_ms": 89.45155464946347,
|
|
"ttft_ms": 342.38094211127645
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "e6cebfc858981b2ae4632d2c75f5e8c1f83280e7306351e15c7cd611d2664d80"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 23.83781385421753,
|
|
"offered_request_rate": 0.4,
|
|
"request_metrics_sha256": "e62567aa33ec429331fcd9f24053a0c79db61ef288f7994e6488ae0f0fa79b7e",
|
|
"request_rate_per_gpu": 0.1,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12394.3405299904,
|
|
"request_id": 0,
|
|
"tpot_ms": 95.48631545343278,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 12520.094438712134,
|
|
"request_id": 1,
|
|
"tpot_ms": 96.04733720338449,
|
|
"ttft_ms": 322.08261388230494
|
|
},
|
|
{
|
|
"e2e_ms": 12556.880443424032,
|
|
"request_id": 2,
|
|
"tpot_ms": 96.10807445129032,
|
|
"ttft_ms": 351.1549881101628
|
|
},
|
|
{
|
|
"e2e_ms": 12502.892311945501,
|
|
"request_id": 3,
|
|
"tpot_ms": 96.14618123456218,
|
|
"ttft_ms": 292.32729515610424
|
|
},
|
|
{
|
|
"e2e_ms": 12539.549613984442,
|
|
"request_id": 4,
|
|
"tpot_ms": 96.30977229626299,
|
|
"ttft_ms": 308.2085323590427
|
|
},
|
|
{
|
|
"e2e_ms": 12576.335618696337,
|
|
"request_id": 5,
|
|
"tpot_ms": 96.32399459689755,
|
|
"ttft_ms": 343.188304890349
|
|
},
|
|
{
|
|
"e2e_ms": 12522.347487217807,
|
|
"request_id": 6,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 289.200173411821
|
|
},
|
|
{
|
|
"e2e_ms": 12559.133491929704,
|
|
"request_id": 7,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 325.98617812371697
|
|
},
|
|
{
|
|
"e2e_ms": 12505.145360451173,
|
|
"request_id": 8,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 271.99804664518723
|
|
},
|
|
{
|
|
"e2e_ms": 12541.802662490163,
|
|
"request_id": 9,
|
|
"tpot_ms": 96.32399459689792,
|
|
"ttft_ms": 308.6553486841268
|
|
},
|
|
{
|
|
"e2e_ms": 12578.588667202104,
|
|
"request_id": 10,
|
|
"tpot_ms": 96.32399459689827,
|
|
"ttft_ms": 345.4413533960228
|
|
},
|
|
{
|
|
"e2e_ms": 12524.60053572362,
|
|
"request_id": 11,
|
|
"tpot_ms": 96.32399459689864,
|
|
"ttft_ms": 291.453221917493
|
|
},
|
|
{
|
|
"e2e_ms": 12561.386540435564,
|
|
"request_id": 12,
|
|
"tpot_ms": 96.32399459689903,
|
|
"ttft_ms": 328.239226629389
|
|
},
|
|
{
|
|
"e2e_ms": 12507.39840895708,
|
|
"request_id": 13,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 274.2510951508592
|
|
},
|
|
{
|
|
"e2e_ms": 12544.05571099607,
|
|
"request_id": 14,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 310.90839718984853
|
|
},
|
|
{
|
|
"e2e_ms": 12580.841715708011,
|
|
"request_id": 15,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 347.6944019017907
|
|
},
|
|
{
|
|
"e2e_ms": 12526.853584229528,
|
|
"request_id": 16,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 293.7062704233071
|
|
},
|
|
{
|
|
"e2e_ms": 12563.63958894147,
|
|
"request_id": 17,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 330.49227513524926
|
|
},
|
|
{
|
|
"e2e_ms": 12509.651457462987,
|
|
"request_id": 18,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 276.5041436567657
|
|
},
|
|
{
|
|
"e2e_ms": 12546.308759501975,
|
|
"request_id": 19,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 313.161445695755
|
|
},
|
|
{
|
|
"e2e_ms": 12583.094764213918,
|
|
"request_id": 20,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 349.94745040769715
|
|
},
|
|
{
|
|
"e2e_ms": 12529.106632735506,
|
|
"request_id": 21,
|
|
"tpot_ms": 96.32399459689994,
|
|
"ttft_ms": 295.95931892921357
|
|
},
|
|
{
|
|
"e2e_ms": 12565.892637447447,
|
|
"request_id": 22,
|
|
"tpot_ms": 96.32399459689992,
|
|
"ttft_ms": 332.7453236411557
|
|
},
|
|
{
|
|
"e2e_ms": 12511.904505968956,
|
|
"request_id": 23,
|
|
"tpot_ms": 96.32399459689988,
|
|
"ttft_ms": 278.75719216267214
|
|
},
|
|
{
|
|
"e2e_ms": 12548.561808007931,
|
|
"request_id": 24,
|
|
"tpot_ms": 96.32399459689977,
|
|
"ttft_ms": 315.41449420166146
|
|
},
|
|
{
|
|
"e2e_ms": 12585.347812719874,
|
|
"request_id": 25,
|
|
"tpot_ms": 96.32399459689977,
|
|
"ttft_ms": 352.2004989136036
|
|
},
|
|
{
|
|
"e2e_ms": 12531.359681241383,
|
|
"request_id": 26,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 298.2123674351982
|
|
},
|
|
{
|
|
"e2e_ms": 12568.145685953326,
|
|
"request_id": 27,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 334.99837214714034
|
|
},
|
|
{
|
|
"e2e_ms": 12514.157554474836,
|
|
"request_id": 28,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 281.01024066864966
|
|
},
|
|
{
|
|
"e2e_ms": 12550.81485651381,
|
|
"request_id": 29,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 317.66754270762476
|
|
},
|
|
{
|
|
"e2e_ms": 12587.600861225752,
|
|
"request_id": 30,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 354.4535474195669
|
|
},
|
|
{
|
|
"e2e_ms": 12533.612729747261,
|
|
"request_id": 31,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 300.46541594107623
|
|
},
|
|
{
|
|
"e2e_ms": 12570.398734459204,
|
|
"request_id": 32,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 337.2514206530184
|
|
},
|
|
{
|
|
"e2e_ms": 12516.410602980714,
|
|
"request_id": 33,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 283.2632891745277
|
|
},
|
|
{
|
|
"e2e_ms": 12553.067905019689,
|
|
"request_id": 34,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 319.9205912135028
|
|
},
|
|
{
|
|
"e2e_ms": 12589.85390973163,
|
|
"request_id": 35,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 356.70659592544496
|
|
},
|
|
{
|
|
"e2e_ms": 12535.865778253139,
|
|
"request_id": 36,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 302.7184644469543
|
|
},
|
|
{
|
|
"e2e_ms": 12572.651782965082,
|
|
"request_id": 37,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 339.5044691588964
|
|
},
|
|
{
|
|
"e2e_ms": 12518.663651486591,
|
|
"request_id": 38,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 285.51633768040574
|
|
},
|
|
{
|
|
"e2e_ms": 12555.320953525566,
|
|
"request_id": 39,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 322.17363971938084
|
|
},
|
|
{
|
|
"e2e_ms": 12770.519034100516,
|
|
"request_id": 40,
|
|
"tpot_ms": 97.72881409188341,
|
|
"ttft_ms": 358.959644431323
|
|
},
|
|
{
|
|
"e2e_ms": 12538.070601439798,
|
|
"request_id": 41,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 304.9715129528323
|
|
},
|
|
{
|
|
"e2e_ms": 12574.856606151741,
|
|
"request_id": 42,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 341.75751766477447
|
|
},
|
|
{
|
|
"e2e_ms": 12520.86847467325,
|
|
"request_id": 43,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 287.7693861862838
|
|
},
|
|
{
|
|
"e2e_ms": 12557.654479385192,
|
|
"request_id": 44,
|
|
"tpot_ms": 96.32462827684986,
|
|
"ttft_ms": 324.4266882252589
|
|
},
|
|
{
|
|
"e2e_ms": 12503.666347906701,
|
|
"request_id": 45,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 270.51903410051636
|
|
},
|
|
{
|
|
"e2e_ms": 12540.323649945676,
|
|
"request_id": 46,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 307.17633613949147
|
|
},
|
|
{
|
|
"e2e_ms": 12577.109654657448,
|
|
"request_id": 47,
|
|
"tpot_ms": 96.32399459689775,
|
|
"ttft_ms": 343.9623408514336
|
|
},
|
|
{
|
|
"e2e_ms": 12523.121523178786,
|
|
"request_id": 48,
|
|
"tpot_ms": 96.3239945968964,
|
|
"ttft_ms": 289.97420937294294
|
|
},
|
|
{
|
|
"e2e_ms": 12559.907527890544,
|
|
"request_id": 49,
|
|
"tpot_ms": 96.32399459689495,
|
|
"ttft_ms": 326.7602140848851
|
|
},
|
|
{
|
|
"e2e_ms": 12505.919396411884,
|
|
"request_id": 50,
|
|
"tpot_ms": 96.32399459689363,
|
|
"ttft_ms": 272.7720826063944
|
|
},
|
|
{
|
|
"e2e_ms": 12542.576698450688,
|
|
"request_id": 51,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 309.4293846453695
|
|
},
|
|
{
|
|
"e2e_ms": 12579.362703162446,
|
|
"request_id": 52,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 346.2153893571269
|
|
},
|
|
{
|
|
"e2e_ms": 12525.374571683784,
|
|
"request_id": 53,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 292.2272578784657
|
|
},
|
|
{
|
|
"e2e_ms": 12562.160576395541,
|
|
"request_id": 54,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 329.0132625902231
|
|
},
|
|
{
|
|
"e2e_ms": 12508.17244491688,
|
|
"request_id": 55,
|
|
"tpot_ms": 96.32399459689226,
|
|
"ttft_ms": 275.0251311115619
|
|
},
|
|
{
|
|
"e2e_ms": 12544.829746955686,
|
|
"request_id": 56,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 311.6824331503665
|
|
},
|
|
{
|
|
"e2e_ms": 12581.615751667443,
|
|
"request_id": 57,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 348.4684378621239
|
|
},
|
|
{
|
|
"e2e_ms": 12527.627620188781,
|
|
"request_id": 58,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 294.4803063834627
|
|
},
|
|
{
|
|
"e2e_ms": 12564.413624900539,
|
|
"request_id": 59,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 331.2663110952201
|
|
},
|
|
{
|
|
"e2e_ms": 12314.305135768365,
|
|
"request_id": 60,
|
|
"tpot_ms": 94.77973981221895,
|
|
"ttft_ms": 277.2781796165589
|
|
},
|
|
{
|
|
"e2e_ms": 12170.36562563149,
|
|
"request_id": 61,
|
|
"tpot_ms": 93.35771766910335,
|
|
"ttft_ms": 313.93548165536345
|
|
},
|
|
{
|
|
"e2e_ms": 12023.933572014499,
|
|
"request_id": 62,
|
|
"tpot_ms": 91.91505579249903,
|
|
"ttft_ms": 350.72148636712086
|
|
},
|
|
{
|
|
"e2e_ms": 11724.699298497455,
|
|
"request_id": 63,
|
|
"tpot_ms": 89.98398380794484,
|
|
"ttft_ms": 296.73335488845964
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "021442528b6c4067d63827496b66b6f530adbe9f4b29732b1aa7aab2112ced9f"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 26.0877902507782,
|
|
"offered_request_rate": 0.8,
|
|
"request_metrics_sha256": "b39b2c6a1c6236e30b246303e0b4d66b80a2515899d8c815efc015ef3f70db54",
|
|
"request_rate_per_gpu": 0.2,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 14028.106647728775,
|
|
"request_id": 0,
|
|
"tpot_ms": 108.35061559310502,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 14267.352384844213,
|
|
"request_id": 1,
|
|
"tpot_ms": 109.65331923259451,
|
|
"ttft_ms": 341.3808423047111
|
|
},
|
|
{
|
|
"e2e_ms": 14406.16288598222,
|
|
"request_id": 2,
|
|
"tpot_ms": 110.64064370419715,
|
|
"ttft_ms": 354.8011355491818
|
|
},
|
|
{
|
|
"e2e_ms": 14444.508775243683,
|
|
"request_id": 3,
|
|
"tpot_ms": 111.53949237897281,
|
|
"ttft_ms": 278.99324311413756
|
|
},
|
|
{
|
|
"e2e_ms": 14585.312621821135,
|
|
"request_id": 4,
|
|
"tpot_ms": 112.58765687327464,
|
|
"ttft_ms": 286.6801989152554
|
|
},
|
|
{
|
|
"e2e_ms": 14727.157204966976,
|
|
"request_id": 5,
|
|
"tpot_ms": 113.57630070139629,
|
|
"ttft_ms": 302.9670158896467
|
|
},
|
|
{
|
|
"e2e_ms": 14871.045643448613,
|
|
"request_id": 6,
|
|
"tpot_ms": 114.43037546464966,
|
|
"ttft_ms": 338.38795943810675
|
|
},
|
|
{
|
|
"e2e_ms": 14913.487485283544,
|
|
"request_id": 7,
|
|
"tpot_ms": 115.12950641852738,
|
|
"ttft_ms": 292.0401701305657
|
|
},
|
|
{
|
|
"e2e_ms": 15059.457396901958,
|
|
"request_id": 8,
|
|
"tpot_ms": 115.80285438725795,
|
|
"ttft_ms": 352.4948897201998
|
|
},
|
|
{
|
|
"e2e_ms": 15271.481983406971,
|
|
"request_id": 9,
|
|
"tpot_ms": 117.58757541921905,
|
|
"ttft_ms": 337.85990516615175
|
|
},
|
|
{
|
|
"e2e_ms": 15316.138289012584,
|
|
"request_id": 10,
|
|
"tpot_ms": 117.88145073243837,
|
|
"ttft_ms": 345.1940459929119
|
|
},
|
|
{
|
|
"e2e_ms": 15090.835365768904,
|
|
"request_id": 11,
|
|
"tpot_ms": 116.63565919716638,
|
|
"ttft_ms": 278.1066477287748
|
|
},
|
|
{
|
|
"e2e_ms": 15302.859952273917,
|
|
"request_id": 12,
|
|
"tpot_ms": 118.0348305674148,
|
|
"ttft_ms": 312.43647021223705
|
|
},
|
|
{
|
|
"e2e_ms": 15347.54563377864,
|
|
"request_id": 13,
|
|
"tpot_ms": 118.10044582475892,
|
|
"ttft_ms": 348.7890140342564
|
|
},
|
|
{
|
|
"e2e_ms": 15289.743982068245,
|
|
"request_id": 14,
|
|
"tpot_ms": 118.15013414242924,
|
|
"ttft_ms": 284.6769459797329
|
|
},
|
|
{
|
|
"e2e_ms": 15334.400287673823,
|
|
"request_id": 15,
|
|
"tpot_ms": 118.19982246009941,
|
|
"ttft_ms": 323.022835241197
|
|
},
|
|
{
|
|
"e2e_ms": 15380.492763872971,
|
|
"request_id": 16,
|
|
"tpot_ms": 118.2528557378034,
|
|
"ttft_ms": 362.38008517193873
|
|
},
|
|
{
|
|
"e2e_ms": 15322.691112162565,
|
|
"request_id": 17,
|
|
"tpot_ms": 118.27848723653601,
|
|
"ttft_ms": 301.32323312249196
|
|
},
|
|
{
|
|
"e2e_ms": 15367.376793667254,
|
|
"request_id": 18,
|
|
"tpot_ms": 118.29615526543176,
|
|
"ttft_ms": 343.765074957421
|
|
},
|
|
{
|
|
"e2e_ms": 15309.575141956848,
|
|
"request_id": 19,
|
|
"tpot_ms": 118.30562851819849,
|
|
"ttft_ms": 284.7603201456401
|
|
},
|
|
{
|
|
"e2e_ms": 15354.260823461536,
|
|
"request_id": 20,
|
|
"tpot_ms": 118.30690699483614,
|
|
"ttft_ms": 329.2836351173456
|
|
},
|
|
{
|
|
"e2e_ms": 15296.45917175113,
|
|
"request_id": 21,
|
|
"tpot_ms": 118.3069069948359,
|
|
"ttft_ms": 271.4819834069715
|
|
},
|
|
{
|
|
"e2e_ms": 15341.115477356709,
|
|
"request_id": 22,
|
|
"tpot_ms": 118.30690699483563,
|
|
"ttft_ms": 316.138289012585
|
|
},
|
|
{
|
|
"e2e_ms": 15387.207953555859,
|
|
"request_id": 23,
|
|
"tpot_ms": 118.31926259505173,
|
|
"ttft_ms": 360.6616039842905
|
|
},
|
|
{
|
|
"e2e_ms": 15329.406301845453,
|
|
"request_id": 24,
|
|
"tpot_ms": 118.31926259505147,
|
|
"ttft_ms": 302.8599522739164
|
|
},
|
|
{
|
|
"e2e_ms": 15375.4987780446,
|
|
"request_id": 25,
|
|
"tpot_ms": 118.33033971862962,
|
|
"ttft_ms": 347.5456337786404
|
|
},
|
|
{
|
|
"e2e_ms": 15317.697126334195,
|
|
"request_id": 26,
|
|
"tpot_ms": 118.33033971862953,
|
|
"ttft_ms": 289.74398206824503
|
|
},
|
|
{
|
|
"e2e_ms": 15362.382807838883,
|
|
"request_id": 27,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 334.400287673823
|
|
},
|
|
{
|
|
"e2e_ms": 15304.581156128477,
|
|
"request_id": 28,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 276.59863596341694
|
|
},
|
|
{
|
|
"e2e_ms": 15349.237461734056,
|
|
"request_id": 29,
|
|
"tpot_ms": 118.3192625950511,
|
|
"ttft_ms": 322.6911121625662
|
|
},
|
|
{
|
|
"e2e_ms": 15395.329937933206,
|
|
"request_id": 30,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 367.37679366725473
|
|
},
|
|
{
|
|
"e2e_ms": 15337.5282862228,
|
|
"request_id": 31,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 309.57514195684865
|
|
},
|
|
{
|
|
"e2e_ms": 15383.620762421948,
|
|
"request_id": 32,
|
|
"tpot_ms": 118.34141684220796,
|
|
"ttft_ms": 354.2608234615372
|
|
},
|
|
{
|
|
"e2e_ms": 15325.819110711542,
|
|
"request_id": 33,
|
|
"tpot_ms": 118.34141684220796,
|
|
"ttft_ms": 296.4591717511311
|
|
},
|
|
{
|
|
"e2e_ms": 15370.50479221623,
|
|
"request_id": 34,
|
|
"tpot_ms": 118.34164814850017,
|
|
"ttft_ms": 341.1154773567091
|
|
},
|
|
{
|
|
"e2e_ms": 15312.703140505824,
|
|
"request_id": 35,
|
|
"tpot_ms": 118.34164814850017,
|
|
"ttft_ms": 283.31382564630303
|
|
},
|
|
{
|
|
"e2e_ms": 15357.388822010513,
|
|
"request_id": 36,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 329.4063018454523
|
|
},
|
|
{
|
|
"e2e_ms": 15299.587170300107,
|
|
"request_id": 37,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 271.6046501350462
|
|
},
|
|
{
|
|
"e2e_ms": 15344.243475905685,
|
|
"request_id": 38,
|
|
"tpot_ms": 118.3192625950511,
|
|
"ttft_ms": 317.69712633419545
|
|
},
|
|
{
|
|
"e2e_ms": 15390.335952104835,
|
|
"request_id": 39,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 362.382807838884
|
|
},
|
|
{
|
|
"e2e_ms": 15332.53430039447,
|
|
"request_id": 40,
|
|
"tpot_ms": 118.33033971862987,
|
|
"ttft_ms": 304.5811561284779
|
|
},
|
|
{
|
|
"e2e_ms": 15378.626776593663,
|
|
"request_id": 41,
|
|
"tpot_ms": 118.34164814850084,
|
|
"ttft_ms": 349.2374617340559
|
|
},
|
|
{
|
|
"e2e_ms": 15320.8251248833,
|
|
"request_id": 42,
|
|
"tpot_ms": 118.34164814850119,
|
|
"ttft_ms": 291.4358100236498
|
|
},
|
|
{
|
|
"e2e_ms": 15365.51080638803,
|
|
"request_id": 43,
|
|
"tpot_ms": 118.33057102492307,
|
|
"ttft_ms": 337.52828622279907
|
|
},
|
|
{
|
|
"e2e_ms": 15307.709154677666,
|
|
"request_id": 44,
|
|
"tpot_ms": 118.33057102492342,
|
|
"ttft_ms": 279.726634512393
|
|
},
|
|
{
|
|
"e2e_ms": 15352.365460283294,
|
|
"request_id": 45,
|
|
"tpot_ms": 118.31926259505316,
|
|
"ttft_ms": 325.81911071154224
|
|
},
|
|
{
|
|
"e2e_ms": 15398.457936482486,
|
|
"request_id": 46,
|
|
"tpot_ms": 118.33033971863193,
|
|
"ttft_ms": 370.5047922162308
|
|
},
|
|
{
|
|
"e2e_ms": 15340.656284772123,
|
|
"request_id": 47,
|
|
"tpot_ms": 118.33033971863227,
|
|
"ttft_ms": 312.7031405058247
|
|
},
|
|
{
|
|
"e2e_ms": 15386.748760971315,
|
|
"request_id": 48,
|
|
"tpot_ms": 118.34141684221105,
|
|
"ttft_ms": 357.38882201051325
|
|
},
|
|
{
|
|
"e2e_ms": 15328.947109260951,
|
|
"request_id": 49,
|
|
"tpot_ms": 118.34141684221137,
|
|
"ttft_ms": 299.5871703001072
|
|
},
|
|
{
|
|
"e2e_ms": 15375.039585460143,
|
|
"request_id": 50,
|
|
"tpot_ms": 118.35272527208234,
|
|
"ttft_ms": 344.24347590568516
|
|
},
|
|
{
|
|
"e2e_ms": 15317.23793374978,
|
|
"request_id": 51,
|
|
"tpot_ms": 118.3527252720827,
|
|
"ttft_ms": 286.4418241952791
|
|
},
|
|
{
|
|
"e2e_ms": 15194.4223437212,
|
|
"request_id": 52,
|
|
"tpot_ms": 117.02274049863567,
|
|
"ttft_ms": 332.53430039447096
|
|
},
|
|
{
|
|
"e2e_ms": 14959.152693280232,
|
|
"request_id": 53,
|
|
"tpot_ms": 115.62535468185925,
|
|
"ttft_ms": 274.7326486841075
|
|
},
|
|
{
|
|
"e2e_ms": 14796.032219342294,
|
|
"request_id": 54,
|
|
"tpot_ms": 113.97800861778735,
|
|
"ttft_ms": 320.8251248832994
|
|
},
|
|
{
|
|
"e2e_ms": 14608.764053703837,
|
|
"request_id": 55,
|
|
"tpot_ms": 112.1516003725654,
|
|
"ttft_ms": 365.5108063880306
|
|
},
|
|
{
|
|
"e2e_ms": 14305.495223453576,
|
|
"request_id": 56,
|
|
"tpot_ms": 110.21878794311739,
|
|
"ttft_ms": 307.70915467766713
|
|
},
|
|
{
|
|
"e2e_ms": 14083.450969580696,
|
|
"request_id": 57,
|
|
"tpot_ms": 108.11878353777482,
|
|
"ttft_ms": 352.36546028329485
|
|
},
|
|
{
|
|
"e2e_ms": 13758.528543574443,
|
|
"request_id": 58,
|
|
"tpot_ms": 106.01547035434261,
|
|
"ttft_ms": 294.5638085729314
|
|
},
|
|
{
|
|
"e2e_ms": 13507.044041669204,
|
|
"request_id": 59,
|
|
"tpot_ms": 103.67234454249669,
|
|
"ttft_ms": 340.6562847721233
|
|
},
|
|
{
|
|
"e2e_ms": 13157.052579489218,
|
|
"request_id": 60,
|
|
"tpot_ms": 101.37163737344456,
|
|
"ttft_ms": 282.85463306175984
|
|
},
|
|
{
|
|
"e2e_ms": 12903.847402123616,
|
|
"request_id": 61,
|
|
"tpot_ms": 99.01496293592649,
|
|
"ttft_ms": 328.9471092609517
|
|
},
|
|
{
|
|
"e2e_ms": 12559.065843040158,
|
|
"request_id": 62,
|
|
"tpot_ms": 96.75527862590212,
|
|
"ttft_ms": 271.1454575505883
|
|
},
|
|
{
|
|
"e2e_ms": 12277.402762692504,
|
|
"request_id": 63,
|
|
"tpot_ms": 94.17452621214743,
|
|
"ttft_ms": 317.23793374978015
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "9ed2bf69d3193253248833822a357db5130e6ca29282e93e04cc92961dba468d"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 25.24283003807068,
|
|
"offered_request_rate": 1.2,
|
|
"request_metrics_sha256": "3825f14be2103486ce094e0b335c86deef683645cae8f05ef3d4beb6fb24cf95",
|
|
"request_rate_per_gpu": 0.3,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 15728.871662360909,
|
|
"request_id": 0,
|
|
"tpot_ms": 121.74246610201946,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 15904.548256233129,
|
|
"request_id": 1,
|
|
"tpot_ms": 122.72470152955445,
|
|
"ttft_ms": 318.5111619797136
|
|
},
|
|
{
|
|
"e2e_ms": 16140.50586206045,
|
|
"request_id": 2,
|
|
"tpot_ms": 124.75532551591714,
|
|
"ttft_ms": 296.5795215389744
|
|
},
|
|
{
|
|
"e2e_ms": 16211.825024161684,
|
|
"request_id": 3,
|
|
"tpot_ms": 125.48331552046938,
|
|
"ttft_ms": 275.44395306207383
|
|
},
|
|
{
|
|
"e2e_ms": 16388.84099440166,
|
|
"request_id": 4,
|
|
"tpot_ms": 126.36635826198417,
|
|
"ttft_ms": 340.31349512967114
|
|
},
|
|
{
|
|
"e2e_ms": 16460.2590055204,
|
|
"request_id": 5,
|
|
"tpot_ms": 127.08834011289514,
|
|
"ttft_ms": 320.03981118271605
|
|
},
|
|
{
|
|
"e2e_ms": 16531.726441149392,
|
|
"request_id": 6,
|
|
"tpot_ms": 127.72809453706097,
|
|
"ttft_ms": 310.25843494264916
|
|
},
|
|
{
|
|
"e2e_ms": 16603.193876778383,
|
|
"request_id": 7,
|
|
"tpot_ms": 128.3170301134443,
|
|
"ttft_ms": 306.93105237095784
|
|
},
|
|
{
|
|
"e2e_ms": 16674.731660081485,
|
|
"request_id": 8,
|
|
"tpot_ms": 128.84921275092722,
|
|
"ttft_ms": 310.8816407137276
|
|
},
|
|
{
|
|
"e2e_ms": 16746.318867894835,
|
|
"request_id": 9,
|
|
"tpot_ms": 129.28898520065795,
|
|
"ttft_ms": 326.61774741127834
|
|
},
|
|
{
|
|
"e2e_ms": 16982.978169021266,
|
|
"request_id": 10,
|
|
"tpot_ms": 130.92468009104587,
|
|
"ttft_ms": 355.5437974584379
|
|
},
|
|
{
|
|
"e2e_ms": 16948.91870099869,
|
|
"request_id": 11,
|
|
"tpot_ms": 131.099598546656,
|
|
"ttft_ms": 299.26968557337295
|
|
},
|
|
{
|
|
"e2e_ms": 17020.55604111586,
|
|
"request_id": 12,
|
|
"tpot_ms": 131.2589874619586,
|
|
"ttft_ms": 350.6646334471153
|
|
},
|
|
{
|
|
"e2e_ms": 16986.496573094286,
|
|
"request_id": 13,
|
|
"tpot_ms": 131.335760528223,
|
|
"ttft_ms": 306.854986009963
|
|
},
|
|
{
|
|
"e2e_ms": 17059.03976122884,
|
|
"request_id": 14,
|
|
"tpot_ms": 131.4398579526774,
|
|
"ttft_ms": 366.1778012388126
|
|
},
|
|
{
|
|
"e2e_ms": 17024.98029320726,
|
|
"request_id": 15,
|
|
"tpot_ms": 131.50690676411472,
|
|
"ttft_ms": 323.6031341646921
|
|
},
|
|
{
|
|
"e2e_ms": 16990.920825185683,
|
|
"request_id": 16,
|
|
"tpot_ms": 131.56847300071718,
|
|
"ttft_ms": 281.7247540946006
|
|
},
|
|
{
|
|
"e2e_ms": 17063.464013320234,
|
|
"request_id": 17,
|
|
"tpot_ms": 131.63886355765484,
|
|
"ttft_ms": 345.32834149806746
|
|
},
|
|
{
|
|
"e2e_ms": 17029.40454529866,
|
|
"request_id": 18,
|
|
"tpot_ms": 131.67812927130495,
|
|
"ttft_ms": 306.28212784293396
|
|
},
|
|
{
|
|
"e2e_ms": 16995.3450772771,
|
|
"request_id": 19,
|
|
"tpot_ms": 131.6904025418972,
|
|
"ttft_ms": 270.66395445615575
|
|
},
|
|
{
|
|
"e2e_ms": 17067.888265411675,
|
|
"request_id": 20,
|
|
"tpot_ms": 131.7034495326512,
|
|
"ttft_ms": 341.5501747649721
|
|
},
|
|
{
|
|
"e2e_ms": 17033.828797390113,
|
|
"request_id": 21,
|
|
"tpot_ms": 131.70595487135952,
|
|
"ttft_ms": 307.17252872745246
|
|
},
|
|
{
|
|
"e2e_ms": 16999.769329368555,
|
|
"request_id": 22,
|
|
"tpot_ms": 131.7084602100679,
|
|
"ttft_ms": 272.79488268993293
|
|
},
|
|
{
|
|
"e2e_ms": 17072.312517503127,
|
|
"request_id": 23,
|
|
"tpot_ms": 131.71809821033827,
|
|
"ttft_ms": 344.1140447901674
|
|
},
|
|
{
|
|
"e2e_ms": 17038.25304948157,
|
|
"request_id": 24,
|
|
"tpot_ms": 131.71982521031822,
|
|
"ttft_ms": 309.83524777115434
|
|
},
|
|
{
|
|
"e2e_ms": 17004.193581460007,
|
|
"request_id": 25,
|
|
"tpot_ms": 131.72116304093396,
|
|
"ttft_ms": 275.60587526139454
|
|
},
|
|
{
|
|
"e2e_ms": 17076.736769594583,
|
|
"request_id": 26,
|
|
"tpot_ms": 131.72963353311178,
|
|
"ttft_ms": 347.07331088938884
|
|
},
|
|
{
|
|
"e2e_ms": 17042.67730157302,
|
|
"request_id": 27,
|
|
"tpot_ms": 131.7305821943633,
|
|
"ttft_ms": 312.8933628888824
|
|
},
|
|
{
|
|
"e2e_ms": 17008.617833551463,
|
|
"request_id": 28,
|
|
"tpot_ms": 131.7309769369132,
|
|
"ttft_ms": 278.783762563485
|
|
},
|
|
{
|
|
"e2e_ms": 17081.161021686035,
|
|
"request_id": 29,
|
|
"tpot_ms": 131.73850434102513,
|
|
"ttft_ms": 350.37097037584175
|
|
},
|
|
{
|
|
"e2e_ms": 17047.986478516996,
|
|
"request_id": 30,
|
|
"tpot_ms": 131.74547225324986,
|
|
"ttft_ms": 316.3115023542638
|
|
},
|
|
{
|
|
"e2e_ms": 17013.927010495438,
|
|
"request_id": 31,
|
|
"tpot_ms": 131.74547225325,
|
|
"ttft_ms": 282.25203433268575
|
|
},
|
|
{
|
|
"e2e_ms": 17086.47019863001,
|
|
"request_id": 32,
|
|
"tpot_ms": 131.75260491481217,
|
|
"ttft_ms": 353.8893744488618
|
|
},
|
|
{
|
|
"e2e_ms": 17053.29565546097,
|
|
"request_id": 33,
|
|
"tpot_ms": 131.7595728270369,
|
|
"ttft_ms": 319.82990642728385
|
|
},
|
|
{
|
|
"e2e_ms": 17019.236187439412,
|
|
"request_id": 34,
|
|
"tpot_ms": 131.75957282703703,
|
|
"ttft_ms": 285.7704384057058
|
|
},
|
|
{
|
|
"e2e_ms": 17091.779375573984,
|
|
"request_id": 35,
|
|
"tpot_ms": 131.7595728270372,
|
|
"ttft_ms": 358.31362654026225
|
|
},
|
|
{
|
|
"e2e_ms": 17058.604832404944,
|
|
"request_id": 36,
|
|
"tpot_ms": 131.7665407392619,
|
|
"ttft_ms": 324.2541585186842
|
|
},
|
|
{
|
|
"e2e_ms": 17024.545364383386,
|
|
"request_id": 37,
|
|
"tpot_ms": 131.76654073926207,
|
|
"ttft_ms": 290.1946904971062
|
|
},
|
|
{
|
|
"e2e_ms": 17097.08855251796,
|
|
"request_id": 38,
|
|
"tpot_ms": 131.76654073926218,
|
|
"ttft_ms": 362.7378786316626
|
|
},
|
|
{
|
|
"e2e_ms": 17063.934932514778,
|
|
"request_id": 39,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 328.6784106101024
|
|
},
|
|
{
|
|
"e2e_ms": 17029.875464493216,
|
|
"request_id": 40,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 294.61894258854215
|
|
},
|
|
{
|
|
"e2e_ms": 17102.41865262779,
|
|
"request_id": 41,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 367.1621307231163
|
|
},
|
|
{
|
|
"e2e_ms": 17069.265032624608,
|
|
"request_id": 42,
|
|
"tpot_ms": 131.78080606238623,
|
|
"ttft_ms": 333.1026627015561
|
|
},
|
|
{
|
|
"e2e_ms": 17035.205564603046,
|
|
"request_id": 43,
|
|
"tpot_ms": 131.78080606238623,
|
|
"ttft_ms": 299.04319467999585
|
|
},
|
|
{
|
|
"e2e_ms": 16941.812657737883,
|
|
"request_id": 44,
|
|
"tpot_ms": 130.47422263719145,
|
|
"ttft_ms": 371.58638281457
|
|
},
|
|
{
|
|
"e2e_ms": 16742.48916517699,
|
|
"request_id": 45,
|
|
"tpot_ms": 129.1729311053857,
|
|
"ttft_ms": 337.5269147930098
|
|
},
|
|
{
|
|
"e2e_ms": 16538.03643831958,
|
|
"request_id": 46,
|
|
"tpot_ms": 127.83125190195379,
|
|
"ttft_ms": 303.46744677144954
|
|
},
|
|
{
|
|
"e2e_ms": 16435.957848641778,
|
|
"request_id": 47,
|
|
"tpot_ms": 126.45627727350987,
|
|
"ttft_ms": 376.0106349060237
|
|
},
|
|
{
|
|
"e2e_ms": 16228.476901947985,
|
|
"request_id": 48,
|
|
"tpot_ms": 125.09075381939779,
|
|
"ttft_ms": 341.9511668844635
|
|
},
|
|
{
|
|
"e2e_ms": 16020.052127617348,
|
|
"request_id": 49,
|
|
"tpot_ms": 123.7177986516098,
|
|
"ttft_ms": 307.89169886290324
|
|
},
|
|
{
|
|
"e2e_ms": 15809.427558622445,
|
|
"request_id": 50,
|
|
"tpot_ms": 122.32055435377619,
|
|
"ttft_ms": 274.71715569286914
|
|
},
|
|
{
|
|
"e2e_ms": 15703.511289947706,
|
|
"request_id": 51,
|
|
"tpot_ms": 120.91536178046667,
|
|
"ttft_ms": 347.26034382843807
|
|
},
|
|
{
|
|
"e2e_ms": 15485.101955905271,
|
|
"request_id": 52,
|
|
"tpot_ms": 119.46378803227081,
|
|
"ttft_ms": 313.20087580687783
|
|
},
|
|
{
|
|
"e2e_ms": 15260.624457846197,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.95746555282956,
|
|
"ttft_ms": 280.02633263684373
|
|
},
|
|
{
|
|
"e2e_ms": 15118.941732007286,
|
|
"request_id": 54,
|
|
"tpot_ms": 116.27064733255806,
|
|
"ttft_ms": 352.56952077241266
|
|
},
|
|
{
|
|
"e2e_ms": 14865.280308326035,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.54149807539514,
|
|
"ttft_ms": 318.5100527508524
|
|
},
|
|
{
|
|
"e2e_ms": 14599.985676841925,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.71378084457564,
|
|
"ttft_ms": 285.3355095808183
|
|
},
|
|
{
|
|
"e2e_ms": 14420.806000135271,
|
|
"request_id": 57,
|
|
"tpot_ms": 110.73171104266838,
|
|
"ttft_ms": 357.87869771638725
|
|
},
|
|
{
|
|
"e2e_ms": 14142.519211198518,
|
|
"request_id": 58,
|
|
"tpot_ms": 108.80866127168261,
|
|
"ttft_ms": 323.819229694827
|
|
},
|
|
{
|
|
"e2e_ms": 13853.830695007076,
|
|
"request_id": 59,
|
|
"tpot_ms": 106.79657547493251,
|
|
"ttft_ms": 290.6656096906488
|
|
},
|
|
{
|
|
"e2e_ms": 13650.503338148035,
|
|
"request_id": 60,
|
|
"tpot_ms": 104.62436645922692,
|
|
"ttft_ms": 363.20879782621773
|
|
},
|
|
{
|
|
"e2e_ms": 13360.876271706531,
|
|
"request_id": 61,
|
|
"tpot_ms": 102.61202316458169,
|
|
"ttft_ms": 329.1493298046575
|
|
},
|
|
{
|
|
"e2e_ms": 13070.674002922458,
|
|
"request_id": 62,
|
|
"tpot_ms": 100.58801805607857,
|
|
"ttft_ms": 295.9957098004793
|
|
},
|
|
{
|
|
"e2e_ms": 12853.555073004585,
|
|
"request_id": 63,
|
|
"tpot_ms": 98.30721397691761,
|
|
"ttft_ms": 368.5388979360482
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.1875,
|
|
"passed": 12
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "3e07b28ac20428d11b8d83064d68561f03804f8adf37111de43398fdd8d94138"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 29.54408025741577,
|
|
"offered_request_rate": 1.6,
|
|
"request_metrics_sha256": "b7d73a1cbef8ca659f753135a4944cb3023d3a2424a19a741ae2198f39c2d172",
|
|
"request_rate_per_gpu": 0.4,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 17513.570713343794,
|
|
"request_id": 0,
|
|
"tpot_ms": 135.79521453495556,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 17840.282304762335,
|
|
"request_id": 1,
|
|
"tpot_ms": 137.71064841138914,
|
|
"ttft_ms": 351.0299565159147
|
|
},
|
|
{
|
|
"e2e_ms": 17895.63618825912,
|
|
"request_id": 2,
|
|
"tpot_ms": 138.10245652748367,
|
|
"ttft_ms": 356.6242092686922
|
|
},
|
|
{
|
|
"e2e_ms": 17848.77057711112,
|
|
"request_id": 3,
|
|
"tpot_ms": 138.39719968518088,
|
|
"ttft_ms": 272.32621709314975
|
|
},
|
|
{
|
|
"e2e_ms": 17904.1244606079,
|
|
"request_id": 4,
|
|
"tpot_ms": 138.80479077491768,
|
|
"ttft_ms": 275.9160321933569
|
|
},
|
|
{
|
|
"e2e_ms": 17959.478344104686,
|
|
"request_id": 5,
|
|
"tpot_ms": 139.18735062645055,
|
|
"ttft_ms": 282.6848145454632
|
|
},
|
|
{
|
|
"e2e_ms": 18014.921946878916,
|
|
"request_id": 6,
|
|
"tpot_ms": 139.5152186600215,
|
|
"ttft_ms": 296.48917705618504
|
|
},
|
|
{
|
|
"e2e_ms": 18070.36554965315,
|
|
"request_id": 7,
|
|
"tpot_ms": 139.8086106441042,
|
|
"ttft_ms": 314.671997851919
|
|
},
|
|
{
|
|
"e2e_ms": 18125.898871704834,
|
|
"request_id": 8,
|
|
"tpot_ms": 140.0646556798527,
|
|
"ttft_ms": 337.68760036353916
|
|
},
|
|
{
|
|
"e2e_ms": 18079.30241838919,
|
|
"request_id": 9,
|
|
"tpot_ms": 140.21470125635153,
|
|
"ttft_ms": 272.0353588325457
|
|
},
|
|
{
|
|
"e2e_ms": 18134.83860521682,
|
|
"request_id": 10,
|
|
"tpot_ms": 140.33922237580643,
|
|
"ttft_ms": 311.75736348940484
|
|
},
|
|
{
|
|
"e2e_ms": 18190.4645113219,
|
|
"request_id": 11,
|
|
"tpot_ms": 140.37869703173442,
|
|
"ttft_ms": 362.36998829163
|
|
},
|
|
{
|
|
"e2e_ms": 18143.870922782207,
|
|
"request_id": 12,
|
|
"tpot_ms": 140.38698051363622,
|
|
"ttft_ms": 314.7243975504068
|
|
},
|
|
{
|
|
"e2e_ms": 18097.274469466563,
|
|
"request_id": 13,
|
|
"tpot_ms": 140.36074521133403,
|
|
"ttft_ms": 271.4598276271438
|
|
},
|
|
{
|
|
"e2e_ms": 18152.810656294194,
|
|
"request_id": 14,
|
|
"tpot_ms": 140.32455966134754,
|
|
"ttft_ms": 331.5915793030584
|
|
},
|
|
{
|
|
"e2e_ms": 18106.217067754496,
|
|
"request_id": 15,
|
|
"tpot_ms": 140.29231877139563,
|
|
"ttft_ms": 289.0925837872533
|
|
},
|
|
{
|
|
"e2e_ms": 18161.753254582127,
|
|
"request_id": 16,
|
|
"tpot_ms": 140.24009333090902,
|
|
"ttft_ms": 351.2614015566804
|
|
},
|
|
{
|
|
"e2e_ms": 18115.15966604243,
|
|
"request_id": 17,
|
|
"tpot_ms": 140.19912163029534,
|
|
"ttft_ms": 309.87121899492286
|
|
},
|
|
{
|
|
"e2e_ms": 18170.69585287006,
|
|
"request_id": 18,
|
|
"tpot_ms": 140.12856604535176,
|
|
"ttft_ms": 374.3679651103839
|
|
},
|
|
{
|
|
"e2e_ms": 18124.102264330366,
|
|
"request_id": 19,
|
|
"tpot_ms": 140.05569589908265,
|
|
"ttft_ms": 337.0288851468697
|
|
},
|
|
{
|
|
"e2e_ms": 18077.42933293013,
|
|
"request_id": 20,
|
|
"tpot_ms": 139.98073600581233,
|
|
"ttft_ms": 299.87586019196044
|
|
},
|
|
{
|
|
"e2e_ms": 18132.96551975776,
|
|
"request_id": 21,
|
|
"tpot_ms": 139.8495015438229,
|
|
"ttft_ms": 372.0788236922541
|
|
},
|
|
{
|
|
"e2e_ms": 18086.369066442116,
|
|
"request_id": 22,
|
|
"tpot_ms": 139.76199603449132,
|
|
"ttft_ms": 336.5955700617178
|
|
},
|
|
{
|
|
"e2e_ms": 18039.682893849018,
|
|
"request_id": 23,
|
|
"tpot_ms": 139.67091992877667,
|
|
"ttft_ms": 301.4760628943822
|
|
},
|
|
{
|
|
"e2e_ms": 18095.21908067664,
|
|
"request_id": 24,
|
|
"tpot_ms": 139.54200325642663,
|
|
"ttft_ms": 373.38466711045726
|
|
},
|
|
{
|
|
"e2e_ms": 18048.532908083544,
|
|
"request_id": 25,
|
|
"tpot_ms": 139.5517742330953,
|
|
"ttft_ms": 325.4575804804407
|
|
},
|
|
{
|
|
"e2e_ms": 18001.846735490446,
|
|
"request_id": 26,
|
|
"tpot_ms": 139.5610841122856,
|
|
"ttft_ms": 277.589053230173
|
|
},
|
|
{
|
|
"e2e_ms": 18057.290338264673,
|
|
"request_id": 27,
|
|
"tpot_ms": 139.56778701592805,
|
|
"ttft_ms": 332.18138724181046
|
|
},
|
|
{
|
|
"e2e_ms": 18010.60416567157,
|
|
"request_id": 28,
|
|
"tpot_ms": 139.57154350494935,
|
|
"ttft_ms": 285.01814054300564
|
|
},
|
|
{
|
|
"e2e_ms": 18066.06100963866,
|
|
"request_id": 29,
|
|
"tpot_ms": 139.5730606683175,
|
|
"ttft_ms": 340.2823047623365
|
|
},
|
|
{
|
|
"e2e_ms": 18019.37483704556,
|
|
"request_id": 30,
|
|
"tpot_ms": 139.57447357032456,
|
|
"ttft_ms": 293.41669361433986
|
|
},
|
|
{
|
|
"e2e_ms": 18074.831681012645,
|
|
"request_id": 31,
|
|
"tpot_ms": 139.57528428268915,
|
|
"ttft_ms": 348.7705771111216
|
|
},
|
|
{
|
|
"e2e_ms": 18028.145508419548,
|
|
"request_id": 32,
|
|
"tpot_ms": 139.57669718469626,
|
|
"ttft_ms": 301.904965963125
|
|
},
|
|
{
|
|
"e2e_ms": 18083.678830471228,
|
|
"request_id": 33,
|
|
"tpot_ms": 139.5781100867033,
|
|
"ttft_ms": 357.25884945990674
|
|
},
|
|
{
|
|
"e2e_ms": 18036.99265787813,
|
|
"request_id": 34,
|
|
"tpot_ms": 139.57881653770684,
|
|
"ttft_ms": 310.48295758936104
|
|
},
|
|
{
|
|
"e2e_ms": 18092.528844705754,
|
|
"request_id": 35,
|
|
"tpot_ms": 139.57954554600127,
|
|
"ttft_ms": 365.9265603635937
|
|
},
|
|
{
|
|
"e2e_ms": 17876.05367572695,
|
|
"request_id": 36,
|
|
"tpot_ms": 138.24262431461773,
|
|
"ttft_ms": 319.240387770499
|
|
},
|
|
{
|
|
"e2e_ms": 17659.21004164318,
|
|
"request_id": 37,
|
|
"tpot_ms": 136.90209533219152,
|
|
"ttft_ms": 272.64393445485524
|
|
},
|
|
{
|
|
"e2e_ms": 17543.75228976072,
|
|
"request_id": 38,
|
|
"tpot_ms": 135.55570892326125,
|
|
"ttft_ms": 328.1772565065388
|
|
},
|
|
{
|
|
"e2e_ms": 17326.367174433202,
|
|
"request_id": 39,
|
|
"tpot_ms": 134.21089375170362,
|
|
"ttft_ms": 281.5836679668422
|
|
},
|
|
{
|
|
"e2e_ms": 17233.832476815856,
|
|
"request_id": 40,
|
|
"tpot_ms": 133.04427482475538,
|
|
"ttft_ms": 337.20957407192384
|
|
},
|
|
{
|
|
"e2e_ms": 17034.265611203216,
|
|
"request_id": 41,
|
|
"tpot_ms": 131.83976083205505,
|
|
"ttft_ms": 290.6159855322272
|
|
},
|
|
{
|
|
"e2e_ms": 16940.502121615402,
|
|
"request_id": 42,
|
|
"tpot_ms": 130.66419538607474,
|
|
"ttft_ms": 346.14930758391085
|
|
},
|
|
{
|
|
"e2e_ms": 16741.97064984836,
|
|
"request_id": 43,
|
|
"tpot_ms": 129.46783410082003,
|
|
"ttft_ms": 299.5557190442142
|
|
},
|
|
{
|
|
"e2e_ms": 16645.559306371404,
|
|
"request_id": 44,
|
|
"tpot_ms": 128.27139685432724,
|
|
"ttft_ms": 355.0919058718449
|
|
},
|
|
{
|
|
"e2e_ms": 16443.232533552808,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.04515130882409,
|
|
"ttft_ms": 308.4983173321483
|
|
},
|
|
{
|
|
"e2e_ms": 16342.299705615802,
|
|
"request_id": 46,
|
|
"tpot_ms": 125.81311182248837,
|
|
"ttft_ms": 364.034504159779
|
|
},
|
|
{
|
|
"e2e_ms": 16135.159559133626,
|
|
"request_id": 47,
|
|
"tpot_ms": 124.54896569695704,
|
|
"ttft_ms": 317.44091562008236
|
|
},
|
|
{
|
|
"e2e_ms": 16031.7032146663,
|
|
"request_id": 48,
|
|
"tpot_ms": 123.29695174036006,
|
|
"ttft_ms": 372.9903436405735
|
|
},
|
|
{
|
|
"e2e_ms": 15823.308920667878,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.0236594458299,
|
|
"ttft_ms": 326.3041710474788
|
|
},
|
|
{
|
|
"e2e_ms": 15613.44809689394,
|
|
"request_id": 50,
|
|
"tpot_ms": 120.73809066445794,
|
|
"ttft_ms": 279.71058250778214
|
|
},
|
|
{
|
|
"e2e_ms": 15508.031714506984,
|
|
"request_id": 51,
|
|
"tpot_ms": 119.47077015706711,
|
|
"ttft_ms": 335.2439045594622
|
|
},
|
|
{
|
|
"e2e_ms": 15292.981047367362,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.14506547559841,
|
|
"ttft_ms": 288.5577319663639
|
|
},
|
|
{
|
|
"e2e_ms": 15175.360910096466,
|
|
"request_id": 53,
|
|
"tpot_ms": 116.78162985277542,
|
|
"ttft_ms": 344.09391879398754
|
|
},
|
|
{
|
|
"e2e_ms": 14945.589828664517,
|
|
"request_id": 54,
|
|
"tpot_ms": 115.34001639735139,
|
|
"ttft_ms": 297.4077462008893
|
|
},
|
|
{
|
|
"e2e_ms": 14803.64975337431,
|
|
"request_id": 55,
|
|
"tpot_ms": 113.78581420786767,
|
|
"ttft_ms": 352.85134897511483
|
|
},
|
|
{
|
|
"e2e_ms": 14557.342221274233,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.21399247946627,
|
|
"ttft_ms": 306.16517638201657
|
|
},
|
|
{
|
|
"e2e_ms": 14399.594833150197,
|
|
"request_id": 57,
|
|
"tpot_ms": 110.5352189984338,
|
|
"ttft_ms": 361.6220203491025
|
|
},
|
|
{
|
|
"e2e_ms": 14144.625862747695,
|
|
"request_id": 58,
|
|
"tpot_ms": 108.89519696843851,
|
|
"ttft_ms": 314.93584775600425
|
|
},
|
|
{
|
|
"e2e_ms": 13973.496543699825,
|
|
"request_id": 59,
|
|
"tpot_ms": 107.1110539525727,
|
|
"ttft_ms": 370.3926917230902
|
|
},
|
|
{
|
|
"e2e_ms": 13708.4999588278,
|
|
"request_id": 60,
|
|
"tpot_ms": 105.3920743283292,
|
|
"ttft_ms": 323.70651912999193
|
|
},
|
|
{
|
|
"e2e_ms": 13445.970803422128,
|
|
"request_id": 61,
|
|
"tpot_ms": 103.6925232825609,
|
|
"ttft_ms": 277.02034653689367
|
|
},
|
|
{
|
|
"e2e_ms": 13273.580023880399,
|
|
"request_id": 62,
|
|
"tpot_ms": 101.89784531725846,
|
|
"ttft_ms": 332.5536685885737
|
|
},
|
|
{
|
|
"e2e_ms": 13000.702540117643,
|
|
"request_id": 63,
|
|
"tpot_ms": 100.1168113710407,
|
|
"ttft_ms": 285.86749599547545
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.203125,
|
|
"passed": 13
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "850087527f8ec5b333ae4bb00c5837c85f7dafca77de3d8c80820e071da8dfbb"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 33.14970922470093,
|
|
"offered_request_rate": 2.4,
|
|
"request_metrics_sha256": "3b42135fbabbd54647284962ccc9fdf0b900fbc0b6ed0bdecf03477d5c409d18",
|
|
"request_rate_per_gpu": 0.6,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 21942.173036302996,
|
|
"request_id": 0,
|
|
"tpot_ms": 170.66609896770518,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 22008.341136862466,
|
|
"request_id": 1,
|
|
"tpot_ms": 170.96613901739173,
|
|
"ttft_ms": 295.64148165371654
|
|
},
|
|
{
|
|
"e2e_ms": 22074.54102422534,
|
|
"request_id": 2,
|
|
"tpot_ms": 171.22660942224667,
|
|
"ttft_ms": 328.7616276000109
|
|
},
|
|
{
|
|
"e2e_ms": 22036.227907899276,
|
|
"request_id": 3,
|
|
"tpot_ms": 171.37344968894675,
|
|
"ttft_ms": 271.7997974030406
|
|
},
|
|
{
|
|
"e2e_ms": 22102.42779526115,
|
|
"request_id": 4,
|
|
"tpot_ms": 171.64131409153669,
|
|
"ttft_ms": 303.98090563598765
|
|
},
|
|
{
|
|
"e2e_ms": 22168.65946942643,
|
|
"request_id": 5,
|
|
"tpot_ms": 171.89659303259273,
|
|
"ttft_ms": 337.79215428715537
|
|
},
|
|
{
|
|
"e2e_ms": 22130.37813990278,
|
|
"request_id": 6,
|
|
"tpot_ms": 172.0281077294899,
|
|
"ttft_ms": 282.80845825756097
|
|
},
|
|
{
|
|
"e2e_ms": 22365.99994575454,
|
|
"request_id": 7,
|
|
"tpot_ms": 173.57085404199856,
|
|
"ttft_ms": 322.5014824207202
|
|
},
|
|
{
|
|
"e2e_ms": 22158.3592864644,
|
|
"request_id": 8,
|
|
"tpot_ms": 172.3496956452424,
|
|
"ttft_ms": 269.9479395186155
|
|
},
|
|
{
|
|
"e2e_ms": 22393.981092316164,
|
|
"request_id": 9,
|
|
"tpot_ms": 173.8419366523574,
|
|
"ttft_ms": 316.0551374667744
|
|
},
|
|
{
|
|
"e2e_ms": 22460.243568400445,
|
|
"request_id": 10,
|
|
"tpot_ms": 173.96590922572528,
|
|
"ttft_ms": 366.57309673333316
|
|
},
|
|
{
|
|
"e2e_ms": 22252.602909110316,
|
|
"request_id": 11,
|
|
"tpot_ms": 172.6883751510306,
|
|
"ttft_ms": 321.1792649294285
|
|
},
|
|
{
|
|
"e2e_ms": 22044.894176229613,
|
|
"request_id": 12,
|
|
"tpot_ms": 171.40112664047544,
|
|
"ttft_ms": 276.9510928892318
|
|
},
|
|
{
|
|
"e2e_ms": 21941.653543442477,
|
|
"request_id": 13,
|
|
"tpot_ms": 170.1065182287643,
|
|
"ttft_ms": 338.1257283894126
|
|
},
|
|
{
|
|
"e2e_ms": 21734.138079618475,
|
|
"request_id": 14,
|
|
"tpot_ms": 168.80971544527205,
|
|
"ttft_ms": 295.30421806892537
|
|
},
|
|
{
|
|
"e2e_ms": 21631.119179231846,
|
|
"request_id": 15,
|
|
"tpot_ms": 167.51267286797687,
|
|
"ttft_ms": 357.0097249987798
|
|
},
|
|
{
|
|
"e2e_ms": 21420.354658102242,
|
|
"request_id": 16,
|
|
"tpot_ms": 166.18662353135022,
|
|
"ttft_ms": 314.65346962076256
|
|
},
|
|
{
|
|
"e2e_ms": 21209.648295563282,
|
|
"request_id": 17,
|
|
"tpot_ms": 164.85776701341018,
|
|
"ttft_ms": 272.7118848601879
|
|
},
|
|
{
|
|
"e2e_ms": 21101.69867612793,
|
|
"request_id": 18,
|
|
"tpot_ms": 163.50974377121332,
|
|
"ttft_ms": 335.9612171838391
|
|
},
|
|
{
|
|
"e2e_ms": 20890.778369055995,
|
|
"request_id": 19,
|
|
"tpot_ms": 162.16628083276893,
|
|
"ttft_ms": 295.66070329433813
|
|
},
|
|
{
|
|
"e2e_ms": 20782.558056179907,
|
|
"request_id": 20,
|
|
"tpot_ms": 160.7985342727971,
|
|
"ttft_ms": 361.14420353467835
|
|
},
|
|
{
|
|
"e2e_ms": 20571.512469390585,
|
|
"request_id": 21,
|
|
"tpot_ms": 159.44598375979533,
|
|
"ttft_ms": 321.8725318965792
|
|
},
|
|
{
|
|
"e2e_ms": 20360.405278761264,
|
|
"request_id": 22,
|
|
"tpot_ms": 158.09565605925098,
|
|
"ttft_ms": 282.2569592363866
|
|
},
|
|
{
|
|
"e2e_ms": 20251.586130359967,
|
|
"request_id": 23,
|
|
"tpot_ms": 156.71227679254048,
|
|
"ttft_ms": 349.12697770732314
|
|
},
|
|
{
|
|
"e2e_ms": 20042.10448060375,
|
|
"request_id": 24,
|
|
"tpot_ms": 155.37142770924103,
|
|
"ttft_ms": 309.93316153013774
|
|
},
|
|
{
|
|
"e2e_ms": 19934.008109113514,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.05960281316982,
|
|
"ttft_ms": 368.43855184094565
|
|
},
|
|
{
|
|
"e2e_ms": 19722.984661067087,
|
|
"request_id": 26,
|
|
"tpot_ms": 152.73931080974432,
|
|
"ttft_ms": 325.0921882295579
|
|
},
|
|
{
|
|
"e2e_ms": 19511.768043076878,
|
|
"request_id": 27,
|
|
"tpot_ms": 151.41621155599998,
|
|
"ttft_ms": 281.9091754648788
|
|
},
|
|
{
|
|
"e2e_ms": 19403.203575383635,
|
|
"request_id": 28,
|
|
"tpot_ms": 150.09414530000853,
|
|
"ttft_ms": 341.2471222825513
|
|
},
|
|
{
|
|
"e2e_ms": 19191.714653347688,
|
|
"request_id": 29,
|
|
"tpot_ms": 148.76487274343572,
|
|
"ttft_ms": 298.57581493135046
|
|
},
|
|
{
|
|
"e2e_ms": 19082.708845359608,
|
|
"request_id": 30,
|
|
"tpot_ms": 147.43434256455518,
|
|
"ttft_ms": 358.5473396611025
|
|
},
|
|
{
|
|
"e2e_ms": 18871.078709760895,
|
|
"request_id": 31,
|
|
"tpot_ms": 146.10277278695904,
|
|
"ttft_ms": 316.0265658170971
|
|
},
|
|
{
|
|
"e2e_ms": 18659.370447496967,
|
|
"request_id": 32,
|
|
"tpot_ms": 144.7695321688903,
|
|
"ttft_ms": 273.6398620479008
|
|
},
|
|
{
|
|
"e2e_ms": 18549.637648096734,
|
|
"request_id": 33,
|
|
"tpot_ms": 143.43132538741386,
|
|
"ttft_ms": 333.85932389517234
|
|
},
|
|
{
|
|
"e2e_ms": 18337.808235788358,
|
|
"request_id": 34,
|
|
"tpot_ms": 142.0958670025312,
|
|
"ttft_ms": 291.63312646689656
|
|
},
|
|
{
|
|
"e2e_ms": 18227.80864738452,
|
|
"request_id": 35,
|
|
"tpot_ms": 140.75352098043697,
|
|
"ttft_ms": 352.1114828690255
|
|
},
|
|
{
|
|
"e2e_ms": 18015.40726100407,
|
|
"request_id": 36,
|
|
"tpot_ms": 139.41149279948888,
|
|
"ttft_ms": 310.14767546898446
|
|
},
|
|
{
|
|
"e2e_ms": 17904.857868774245,
|
|
"request_id": 37,
|
|
"tpot_ms": 138.06211837292886,
|
|
"ttft_ms": 370.9688354122758
|
|
},
|
|
{
|
|
"e2e_ms": 17692.008101355255,
|
|
"request_id": 38,
|
|
"tpot_ms": 136.71510258553306,
|
|
"ttft_ms": 329.1900729925583
|
|
},
|
|
{
|
|
"e2e_ms": 17479.148877024498,
|
|
"request_id": 39,
|
|
"tpot_ms": 135.36749298254742,
|
|
"ttft_ms": 287.47726824097344
|
|
},
|
|
{
|
|
"e2e_ms": 17381.961391787096,
|
|
"request_id": 40,
|
|
"tpot_ms": 134.1089349239123,
|
|
"ttft_ms": 350.1266564502323
|
|
},
|
|
{
|
|
"e2e_ms": 17178.01129231477,
|
|
"request_id": 41,
|
|
"tpot_ms": 132.8327622110564,
|
|
"ttft_ms": 308.2504915106092
|
|
},
|
|
{
|
|
"e2e_ms": 17080.086531895085,
|
|
"request_id": 42,
|
|
"tpot_ms": 131.5805170059729,
|
|
"ttft_ms": 369.36087213652866
|
|
},
|
|
{
|
|
"e2e_ms": 16876.654129344566,
|
|
"request_id": 43,
|
|
"tpot_ms": 130.30640250229436,
|
|
"ttft_ms": 327.74101155318294
|
|
},
|
|
{
|
|
"e2e_ms": 16671.422925287778,
|
|
"request_id": 44,
|
|
"tpot_ms": 129.0178430005707,
|
|
"ttft_ms": 286.15686421529676
|
|
},
|
|
{
|
|
"e2e_ms": 16571.761179006833,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.74972161297816,
|
|
"ttft_ms": 347.5465341586066
|
|
},
|
|
{
|
|
"e2e_ms": 16364.721381165033,
|
|
"request_id": 46,
|
|
"tpot_ms": 126.44572566423672,
|
|
"ttft_ms": 306.1142218069683
|
|
},
|
|
{
|
|
"e2e_ms": 16261.449604637399,
|
|
"request_id": 47,
|
|
"tpot_ms": 125.14700916940275,
|
|
"ttft_ms": 367.77944012325037
|
|
},
|
|
{
|
|
"e2e_ms": 16053.400400183471,
|
|
"request_id": 48,
|
|
"tpot_ms": 123.8340136000885,
|
|
"ttft_ms": 326.48067297223093
|
|
},
|
|
{
|
|
"e2e_ms": 15845.03658651726,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.50141376236263,
|
|
"ttft_ms": 287.35703869720555
|
|
},
|
|
{
|
|
"e2e_ms": 15739.724302020804,
|
|
"request_id": 50,
|
|
"tpot_ms": 121.15600683324593,
|
|
"ttft_ms": 352.9114341985711
|
|
},
|
|
{
|
|
"e2e_ms": 15530.891082399023,
|
|
"request_id": 51,
|
|
"tpot_ms": 119.81946138926746,
|
|
"ttft_ms": 313.8194859620569
|
|
},
|
|
{
|
|
"e2e_ms": 15319.199082162215,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.45427332697808,
|
|
"ttft_ms": 275.5063696359983
|
|
},
|
|
{
|
|
"e2e_ms": 15206.960333133673,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.04949498375757,
|
|
"ttft_ms": 341.6744701964625
|
|
},
|
|
{
|
|
"e2e_ms": 14987.9081257507,
|
|
"request_id": 54,
|
|
"tpot_ms": 115.62635253449054,
|
|
"ttft_ms": 303.36135387040383
|
|
},
|
|
{
|
|
"e2e_ms": 14861.077413909577,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.10642655651417,
|
|
"ttft_ms": 369.5612412322795
|
|
},
|
|
{
|
|
"e2e_ms": 14633.756981193535,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.61817997075843,
|
|
"ttft_ms": 331.2481249072157
|
|
},
|
|
{
|
|
"e2e_ms": 14403.991359276923,
|
|
"request_id": 57,
|
|
"tpot_ms": 111.11042963695559,
|
|
"ttft_ms": 292.9667953835633
|
|
},
|
|
{
|
|
"e2e_ms": 14264.847964808048,
|
|
"request_id": 58,
|
|
"tpot_ms": 109.49330311228506,
|
|
"ttft_ms": 359.1984695478452
|
|
},
|
|
{
|
|
"e2e_ms": 14029.729570522895,
|
|
"request_id": 59,
|
|
"tpot_ms": 107.94340496454889,
|
|
"ttft_ms": 320.91714002518756
|
|
},
|
|
{
|
|
"e2e_ms": 13793.064611419886,
|
|
"request_id": 60,
|
|
"tpot_ms": 106.38108660628622,
|
|
"ttft_ms": 282.66661242153646
|
|
},
|
|
{
|
|
"e2e_ms": 13648.251078198633,
|
|
"request_id": 61,
|
|
"tpot_ms": 104.71931331978594,
|
|
"ttft_ms": 348.89828658581837
|
|
},
|
|
{
|
|
"e2e_ms": 13412.62809971594,
|
|
"request_id": 62,
|
|
"tpot_ms": 103.16519953332896,
|
|
"ttft_ms": 310.647758983162
|
|
},
|
|
{
|
|
"e2e_ms": 13260.053320226874,
|
|
"request_id": 63,
|
|
"tpot_ms": 101.44207153668842,
|
|
"ttft_ms": 376.9102350674451
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.203125,
|
|
"passed": 13
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.546875,
|
|
"passed": 35
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "0b9f30d883de83b4af69dd114bd5d2bb974b7fd1f01d45f913bf5049e4b72a86"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 36.70769214630127,
|
|
"offered_request_rate": 3.2,
|
|
"request_metrics_sha256": "bfba1ae71cfb6d0595614b14bc49ac47591bfad6792be0a3d98d2c98b16012e8",
|
|
"request_rate_per_gpu": 0.8,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 24003.81066522886,
|
|
"request_id": 0,
|
|
"tpot_ms": 186.89946612460176,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 23901.058156177503,
|
|
"request_id": 1,
|
|
"tpot_ms": 185.74139588389912,
|
|
"ttft_ms": 311.9008789223171
|
|
},
|
|
{
|
|
"e2e_ms": 23798.190109844723,
|
|
"request_id": 2,
|
|
"tpot_ms": 184.56225305811301,
|
|
"ttft_ms": 358.7839714643697
|
|
},
|
|
{
|
|
"e2e_ms": 23590.44639829378,
|
|
"request_id": 3,
|
|
"tpot_ms": 183.2673714648551,
|
|
"ttft_ms": 315.49022225718517
|
|
},
|
|
{
|
|
"e2e_ms": 23382.70131446869,
|
|
"request_id": 4,
|
|
"tpot_ms": 181.9732705315891,
|
|
"ttft_ms": 272.09595695687216
|
|
},
|
|
{
|
|
"e2e_ms": 23279.64954297851,
|
|
"request_id": 5,
|
|
"tpot_ms": 180.78914255292864,
|
|
"ttft_ms": 319.4284387565705
|
|
},
|
|
{
|
|
"e2e_ms": 23071.973865900134,
|
|
"request_id": 6,
|
|
"tpot_ms": 179.49398625037213,
|
|
"ttft_ms": 276.2376121028707
|
|
},
|
|
{
|
|
"e2e_ms": 22969.022726489202,
|
|
"request_id": 7,
|
|
"tpot_ms": 178.2861180199918,
|
|
"ttft_ms": 326.6857379502426
|
|
},
|
|
{
|
|
"e2e_ms": 22761.29942646063,
|
|
"request_id": 8,
|
|
"tpot_ms": 176.98924979786668,
|
|
"ttft_ms": 283.66470213156344
|
|
},
|
|
{
|
|
"e2e_ms": 22658.264587165988,
|
|
"request_id": 9,
|
|
"tpot_ms": 175.7551174352214,
|
|
"ttft_ms": 337.36467289287384
|
|
},
|
|
{
|
|
"e2e_ms": 22450.304224673095,
|
|
"request_id": 10,
|
|
"tpot_ms": 174.4550575162989,
|
|
"ttft_ms": 294.51192010313275
|
|
},
|
|
{
|
|
"e2e_ms": 22346.821408551063,
|
|
"request_id": 11,
|
|
"tpot_ms": 173.1774539652428,
|
|
"ttft_ms": 353.28475496522805
|
|
},
|
|
{
|
|
"e2e_ms": 22138.79812545183,
|
|
"request_id": 12,
|
|
"tpot_ms": 171.87427427697472,
|
|
"ttft_ms": 310.7652922760442
|
|
},
|
|
{
|
|
"e2e_ms": 22035.248814705075,
|
|
"request_id": 13,
|
|
"tpot_ms": 170.57508935568035,
|
|
"ttft_ms": 372.21246653366967
|
|
},
|
|
{
|
|
"e2e_ms": 21827.324416126074,
|
|
"request_id": 14,
|
|
"tpot_ms": 169.27188182433355,
|
|
"ttft_ms": 329.79542443571484
|
|
},
|
|
{
|
|
"e2e_ms": 21619.373671552865,
|
|
"request_id": 15,
|
|
"tpot_ms": 167.96821838814947,
|
|
"ttft_ms": 287.40993625788303
|
|
},
|
|
{
|
|
"e2e_ms": 21512.77581709026,
|
|
"request_id": 16,
|
|
"tpot_ms": 166.64216905152279,
|
|
"ttft_ms": 349.2203475468658
|
|
},
|
|
{
|
|
"e2e_ms": 21303.255969153783,
|
|
"request_id": 17,
|
|
"tpot_ms": 165.32482604305434,
|
|
"ttft_ms": 307.003061685883
|
|
},
|
|
{
|
|
"e2e_ms": 21196.567333974876,
|
|
"request_id": 18,
|
|
"tpot_ms": 163.99164288866731,
|
|
"ttft_ms": 369.62868711412875
|
|
},
|
|
{
|
|
"e2e_ms": 20986.94051377241,
|
|
"request_id": 19,
|
|
"tpot_ms": 162.67002407488405,
|
|
"ttft_ms": 327.84745626213316
|
|
},
|
|
{
|
|
"e2e_ms": 20777.255965035714,
|
|
"request_id": 20,
|
|
"tpot_ms": 161.34765321373814,
|
|
"ttft_ms": 286.10400689097213
|
|
},
|
|
{
|
|
"e2e_ms": 20670.377044913395,
|
|
"request_id": 21,
|
|
"tpot_ms": 159.9951027007364,
|
|
"ttft_ms": 350.99900191987297
|
|
},
|
|
{
|
|
"e2e_ms": 20460.656782932234,
|
|
"request_id": 22,
|
|
"tpot_ms": 158.67196467076286,
|
|
"ttft_ms": 309.3172697453532
|
|
},
|
|
{
|
|
"e2e_ms": 20353.38846177537,
|
|
"request_id": 23,
|
|
"tpot_ms": 157.31793948495124,
|
|
"ttft_ms": 374.01014718656
|
|
},
|
|
{
|
|
"e2e_ms": 20144.48097023076,
|
|
"request_id": 24,
|
|
"tpot_ms": 155.99981541047092,
|
|
"ttft_ms": 332.5044131009545
|
|
},
|
|
{
|
|
"e2e_ms": 19934.83773528968,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.6729368346084,
|
|
"ttft_ms": 291.37475729441144
|
|
},
|
|
{
|
|
"e2e_ms": 19827.980953909257,
|
|
"request_id": 26,
|
|
"tpot_ms": 153.35264483118294,
|
|
"ttft_ms": 352.1950603490236
|
|
},
|
|
{
|
|
"e2e_ms": 19618.20597824765,
|
|
"request_id": 27,
|
|
"tpot_ms": 152.02412197401114,
|
|
"ttft_ms": 311.1424875482367
|
|
},
|
|
{
|
|
"e2e_ms": 19511.107444230154,
|
|
"request_id": 28,
|
|
"tpot_ms": 150.6975599886974,
|
|
"ttft_ms": 372.5173256655836
|
|
},
|
|
{
|
|
"e2e_ms": 19301.196316545185,
|
|
"request_id": 29,
|
|
"tpot_ms": 149.36537894653213,
|
|
"ttft_ms": 331.7931903356044
|
|
},
|
|
{
|
|
"e2e_ms": 19091.249936104825,
|
|
"request_id": 30,
|
|
"tpot_ms": 148.03267778881258,
|
|
"ttft_ms": 291.0998569256247
|
|
},
|
|
{
|
|
"e2e_ms": 18983.78646717311,
|
|
"request_id": 31,
|
|
"tpot_ms": 146.7011080112165,
|
|
"ttft_ms": 352.7457497486193
|
|
},
|
|
{
|
|
"e2e_ms": 18773.765669374145,
|
|
"request_id": 32,
|
|
"tpot_ms": 145.36784483585674,
|
|
"ttft_ms": 312.0493752203348
|
|
},
|
|
{
|
|
"e2e_ms": 18665.888247551986,
|
|
"request_id": 33,
|
|
"tpot_ms": 144.03124523287858,
|
|
"ttft_ms": 373.9201029764097
|
|
},
|
|
{
|
|
"e2e_ms": 18455.8068747313,
|
|
"request_id": 34,
|
|
"tpot_ms": 142.6968275070543,
|
|
"ttft_ms": 333.30978133540157
|
|
},
|
|
{
|
|
"e2e_ms": 18245.529234152025,
|
|
"request_id": 35,
|
|
"tpot_ms": 141.3602386069844,
|
|
"ttft_ms": 292.7789310650048
|
|
},
|
|
{
|
|
"e2e_ms": 18137.294514438578,
|
|
"request_id": 36,
|
|
"tpot_ms": 140.01821042603632,
|
|
"ttft_ms": 354.98179033196385
|
|
},
|
|
{
|
|
"e2e_ms": 17926.83360591763,
|
|
"request_id": 37,
|
|
"tpot_ms": 138.67967407469433,
|
|
"ttft_ms": 314.5149984314486
|
|
},
|
|
{
|
|
"e2e_ms": 17716.242055541137,
|
|
"request_id": 38,
|
|
"tpot_ms": 137.3398665125213,
|
|
"ttft_ms": 274.0790084509328
|
|
},
|
|
{
|
|
"e2e_ms": 17607.549497877386,
|
|
"request_id": 39,
|
|
"tpot_ms": 135.99225690953574,
|
|
"ttft_ms": 336.53287036634794
|
|
},
|
|
{
|
|
"e2e_ms": 17401.54255835392,
|
|
"request_id": 40,
|
|
"tpot_ms": 134.6880568492608,
|
|
"ttft_ms": 296.1593384978016
|
|
},
|
|
{
|
|
"e2e_ms": 17301.7591255476,
|
|
"request_id": 41,
|
|
"tpot_ms": 133.41188413640484,
|
|
"ttft_ms": 358.4498402241838
|
|
},
|
|
{
|
|
"e2e_ms": 17095.506427630044,
|
|
"request_id": 42,
|
|
"tpot_ms": 132.10522228512968,
|
|
"ttft_ms": 318.1431974185731
|
|
},
|
|
{
|
|
"e2e_ms": 16889.623559688287,
|
|
"request_id": 43,
|
|
"tpot_ms": 130.8012299461049,
|
|
"ttft_ms": 277.86735653296194
|
|
},
|
|
{
|
|
"e2e_ms": 16788.559022297493,
|
|
"request_id": 44,
|
|
"tpot_ms": 129.5126704443812,
|
|
"ttft_ms": 340.449875861081
|
|
},
|
|
{
|
|
"e2e_ms": 16581.727329092846,
|
|
"request_id": 45,
|
|
"tpot_ms": 128.200721970688,
|
|
"ttft_ms": 300.23563881547057
|
|
},
|
|
{
|
|
"e2e_ms": 16478.854197918037,
|
|
"request_id": 46,
|
|
"tpot_ms": 126.8967260219465,
|
|
"ttft_ms": 362.9699931308306
|
|
},
|
|
{
|
|
"e2e_ms": 16270.819161297495,
|
|
"request_id": 47,
|
|
"tpot_ms": 125.57481733363996,
|
|
"ttft_ms": 322.8173599252209
|
|
},
|
|
{
|
|
"e2e_ms": 16062.627892404027,
|
|
"request_id": 48,
|
|
"tpot_ms": 124.25122687214136,
|
|
"ttft_ms": 282.72207964207576
|
|
},
|
|
{
|
|
"e2e_ms": 15958.430745404816,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.91862703441548,
|
|
"ttft_ms": 347.7651120340504
|
|
},
|
|
{
|
|
"e2e_ms": 15749.715539461335,
|
|
"request_id": 50,
|
|
"tpot_ms": 121.58574387256766,
|
|
"ttft_ms": 308.3260676452415
|
|
},
|
|
{
|
|
"e2e_ms": 15645.048986506545,
|
|
"request_id": 51,
|
|
"tpot_ms": 120.24919842858912,
|
|
"ttft_ms": 373.40078607572735
|
|
},
|
|
{
|
|
"e2e_ms": 15435.036319721643,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.8997102377205,
|
|
"ttft_ms": 334.77311953113895
|
|
},
|
|
{
|
|
"e2e_ms": 15224.012292267464,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.54201604221191,
|
|
"ttft_ms": 296.1762549065519
|
|
},
|
|
{
|
|
"e2e_ms": 15109.12675155149,
|
|
"request_id": 54,
|
|
"tpot_ms": 116.11789378026353,
|
|
"ttft_ms": 362.1542414580219
|
|
},
|
|
{
|
|
"e2e_ms": 14893.238736493451,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.72141540015762,
|
|
"ttft_ms": 323.6189806734337
|
|
},
|
|
{
|
|
"e2e_ms": 14675.411853468433,
|
|
"request_id": 56,
|
|
"tpot_ms": 113.30942780834319,
|
|
"ttft_ms": 285.1145218088469
|
|
},
|
|
{
|
|
"e2e_ms": 14549.812898218817,
|
|
"request_id": 57,
|
|
"tpot_ms": 111.79839094375423,
|
|
"ttft_ms": 351.4172483620292
|
|
},
|
|
{
|
|
"e2e_ms": 14329.820655618192,
|
|
"request_id": 58,
|
|
"tpot_ms": 110.36886820693505,
|
|
"ttft_ms": 312.97439333744137
|
|
},
|
|
{
|
|
"e2e_ms": 14108.094791808619,
|
|
"request_id": 59,
|
|
"tpot_ms": 108.92545237461232,
|
|
"ttft_ms": 274.56234023285475
|
|
},
|
|
{
|
|
"e2e_ms": 13975.596499372607,
|
|
"request_id": 60,
|
|
"tpot_ms": 107.35951607044248,
|
|
"ttft_ms": 340.93795842641225
|
|
},
|
|
{
|
|
"e2e_ms": 13753.714210521188,
|
|
"request_id": 61,
|
|
"tpot_ms": 105.91438347527057,
|
|
"ttft_ms": 302.58750916182464
|
|
},
|
|
{
|
|
"e2e_ms": 13622.257898704496,
|
|
"request_id": 62,
|
|
"tpot_ms": 104.35568588071192,
|
|
"ttft_ms": 369.08579185408286
|
|
},
|
|
{
|
|
"e2e_ms": 13397.788527763809,
|
|
"request_id": 63,
|
|
"tpot_ms": 102.88969749082138,
|
|
"ttft_ms": 330.7969464294942
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.1875,
|
|
"passed": 12
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.546875,
|
|
"passed": 35
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.90625,
|
|
"passed": 58
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "96c4e04e8f133e66449a5567fcde568f27af44bd9f5d1cadeade66b771a8f0f3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"loads": [
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 32.150320291519165,
|
|
"offered_request_rate": 0.1,
|
|
"request_metrics_sha256": "b45cd3a473a98bee1127b2315d0c0c59ded580275f6e6d10a3e5a52d85d24de4",
|
|
"request_rate_per_gpu": 0.025,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 11656.879780530655,
|
|
"request_id": 0,
|
|
"tpot_ms": 89.67953789863164,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 11729.945143691864,
|
|
"request_id": 1,
|
|
"tpot_ms": 89.99959080602144,
|
|
"ttft_ms": 299.9971113271407
|
|
},
|
|
{
|
|
"e2e_ms": 11712.730146443742,
|
|
"request_id": 2,
|
|
"tpot_ms": 90.00056214193414,
|
|
"ttft_ms": 282.65875441810806
|
|
},
|
|
{
|
|
"e2e_ms": 11785.795509604945,
|
|
"request_id": 3,
|
|
"tpot_ms": 89.99959080602139,
|
|
"ttft_ms": 355.8474772402285
|
|
},
|
|
{
|
|
"e2e_ms": 11770.95360336775,
|
|
"request_id": 4,
|
|
"tpot_ms": 90.01924789792568,
|
|
"ttft_ms": 338.50912033118874
|
|
},
|
|
{
|
|
"e2e_ms": 11756.111697130556,
|
|
"request_id": 5,
|
|
"tpot_ms": 90.0192478979257,
|
|
"ttft_ms": 323.66721409399446
|
|
},
|
|
{
|
|
"e2e_ms": 11741.269790893028,
|
|
"request_id": 6,
|
|
"tpot_ms": 90.01924789792305,
|
|
"ttft_ms": 308.8253078568002
|
|
},
|
|
{
|
|
"e2e_ms": 11726.427884655508,
|
|
"request_id": 7,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 293.98340161927194
|
|
},
|
|
{
|
|
"e2e_ms": 11709.212887407048,
|
|
"request_id": 8,
|
|
"tpot_ms": 90.00056214193147,
|
|
"ttft_ms": 279.1414953817508
|
|
},
|
|
{
|
|
"e2e_ms": 11782.278250567928,
|
|
"request_id": 9,
|
|
"tpot_ms": 89.99959080601886,
|
|
"ttft_ms": 352.3302182035337
|
|
},
|
|
{
|
|
"e2e_ms": 11767.436344330406,
|
|
"request_id": 10,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 334.9918612941707
|
|
},
|
|
{
|
|
"e2e_ms": 11752.594438092885,
|
|
"request_id": 11,
|
|
"tpot_ms": 90.01924789792311,
|
|
"ttft_ms": 320.14995505664956
|
|
},
|
|
{
|
|
"e2e_ms": 11737.752531855705,
|
|
"request_id": 12,
|
|
"tpot_ms": 90.01924789792581,
|
|
"ttft_ms": 305.30804881912843
|
|
},
|
|
{
|
|
"e2e_ms": 11722.91062561817,
|
|
"request_id": 13,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 290.46614258194836
|
|
},
|
|
{
|
|
"e2e_ms": 11705.69562836971,
|
|
"request_id": 14,
|
|
"tpot_ms": 90.00056214193148,
|
|
"ttft_ms": 275.624236344413
|
|
},
|
|
{
|
|
"e2e_ms": 11778.76099153056,
|
|
"request_id": 15,
|
|
"tpot_ms": 89.99959080601862,
|
|
"ttft_ms": 348.8129591661959
|
|
},
|
|
{
|
|
"e2e_ms": 11763.919085293026,
|
|
"request_id": 16,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 331.4746022568045
|
|
},
|
|
{
|
|
"e2e_ms": 11749.07717905549,
|
|
"request_id": 17,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 316.63269601926913
|
|
},
|
|
{
|
|
"e2e_ms": 11734.235272817954,
|
|
"request_id": 18,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 301.7907897817338
|
|
},
|
|
{
|
|
"e2e_ms": 11719.39336658042,
|
|
"request_id": 19,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 286.94888354419845
|
|
},
|
|
{
|
|
"e2e_ms": 11702.178369331961,
|
|
"request_id": 20,
|
|
"tpot_ms": 90.00056214193148,
|
|
"ttft_ms": 272.1069773066631
|
|
},
|
|
{
|
|
"e2e_ms": 11775.24373249281,
|
|
"request_id": 21,
|
|
"tpot_ms": 89.99959080601862,
|
|
"ttft_ms": 345.295700128446
|
|
},
|
|
{
|
|
"e2e_ms": 11760.401826255276,
|
|
"request_id": 22,
|
|
"tpot_ms": 90.01924789792301,
|
|
"ttft_ms": 327.95734321905456
|
|
},
|
|
{
|
|
"e2e_ms": 11745.55992001774,
|
|
"request_id": 23,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 313.1154369815192
|
|
},
|
|
{
|
|
"e2e_ms": 11730.718013780204,
|
|
"request_id": 24,
|
|
"tpot_ms": 90.019247897923,
|
|
"ttft_ms": 298.2735307439839
|
|
},
|
|
{
|
|
"e2e_ms": 11713.50301653041,
|
|
"request_id": 25,
|
|
"tpot_ms": 90.00056214192095,
|
|
"ttft_ms": 283.43162450644854
|
|
},
|
|
{
|
|
"e2e_ms": 11786.568379689925,
|
|
"request_id": 26,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 356.6203473268956
|
|
},
|
|
{
|
|
"e2e_ms": 11771.726473451054,
|
|
"request_id": 27,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 339.28199041616836
|
|
},
|
|
{
|
|
"e2e_ms": 11756.884567212182,
|
|
"request_id": 28,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 324.4400841772972
|
|
},
|
|
{
|
|
"e2e_ms": 11742.042660973311,
|
|
"request_id": 29,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 309.59817793842603
|
|
},
|
|
{
|
|
"e2e_ms": 11727.20075473444,
|
|
"request_id": 30,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 294.7562716995549
|
|
},
|
|
{
|
|
"e2e_ms": 11709.985757484617,
|
|
"request_id": 31,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 279.9143654606837
|
|
},
|
|
{
|
|
"e2e_ms": 11783.051120644132,
|
|
"request_id": 32,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 353.1030882811024
|
|
},
|
|
{
|
|
"e2e_ms": 11768.20921440526,
|
|
"request_id": 33,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 335.7647313703751
|
|
},
|
|
{
|
|
"e2e_ms": 11753.36730816639,
|
|
"request_id": 34,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 320.92282513150394
|
|
},
|
|
{
|
|
"e2e_ms": 11738.525401927518,
|
|
"request_id": 35,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 306.0809188926328
|
|
},
|
|
{
|
|
"e2e_ms": 11723.683495688647,
|
|
"request_id": 36,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 291.2390126537616
|
|
},
|
|
{
|
|
"e2e_ms": 11706.468498438824,
|
|
"request_id": 37,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 276.39710641489046
|
|
},
|
|
{
|
|
"e2e_ms": 11779.533861598338,
|
|
"request_id": 38,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 349.5858292353091
|
|
},
|
|
{
|
|
"e2e_ms": 11764.691955359467,
|
|
"request_id": 39,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 332.24747232458185
|
|
},
|
|
{
|
|
"e2e_ms": 11749.850049120596,
|
|
"request_id": 40,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 317.4055660857107
|
|
},
|
|
{
|
|
"e2e_ms": 11735.008142881725,
|
|
"request_id": 41,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 302.5636598468395
|
|
},
|
|
{
|
|
"e2e_ms": 11720.166236642854,
|
|
"request_id": 42,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 287.72175360796837
|
|
},
|
|
{
|
|
"e2e_ms": 11702.95123939303,
|
|
"request_id": 43,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 272.8798473690972
|
|
},
|
|
{
|
|
"e2e_ms": 11776.016602552545,
|
|
"request_id": 44,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 346.06857018951587
|
|
},
|
|
{
|
|
"e2e_ms": 11761.174696313674,
|
|
"request_id": 45,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 328.7302132787886
|
|
},
|
|
{
|
|
"e2e_ms": 11746.332790074803,
|
|
"request_id": 46,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 313.88830703991744
|
|
},
|
|
{
|
|
"e2e_ms": 11731.490883835932,
|
|
"request_id": 47,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 299.0464008010463
|
|
},
|
|
{
|
|
"e2e_ms": 11716.64897759706,
|
|
"request_id": 48,
|
|
"tpot_ms": 90.01924789791249,
|
|
"ttft_ms": 284.2044945621751
|
|
},
|
|
{
|
|
"e2e_ms": 11699.433980347238,
|
|
"request_id": 49,
|
|
"tpot_ms": 90.00056214192074,
|
|
"ttft_ms": 269.36258832330395
|
|
},
|
|
{
|
|
"e2e_ms": 11772.499343506752,
|
|
"request_id": 50,
|
|
"tpot_ms": 89.9995908060081,
|
|
"ttft_ms": 342.5513111437226
|
|
},
|
|
{
|
|
"e2e_ms": 11757.657437270042,
|
|
"request_id": 51,
|
|
"tpot_ms": 90.0192478979295,
|
|
"ttft_ms": 325.21295423299534
|
|
},
|
|
{
|
|
"e2e_ms": 11742.815531033444,
|
|
"request_id": 52,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 310.3710479967958
|
|
},
|
|
{
|
|
"e2e_ms": 11727.973624796847,
|
|
"request_id": 53,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 295.5291417601984
|
|
},
|
|
{
|
|
"e2e_ms": 11710.758627549352,
|
|
"request_id": 54,
|
|
"tpot_ms": 90.00056214193505,
|
|
"ttft_ms": 280.687235523601
|
|
},
|
|
{
|
|
"e2e_ms": 11783.823990711198,
|
|
"request_id": 55,
|
|
"tpot_ms": 89.99959080602287,
|
|
"ttft_ms": 353.8759583462934
|
|
},
|
|
{
|
|
"e2e_ms": 11768.982084474601,
|
|
"request_id": 56,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 336.5376014379535
|
|
},
|
|
{
|
|
"e2e_ms": 11754.140178238004,
|
|
"request_id": 57,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 321.6956952013561
|
|
},
|
|
{
|
|
"e2e_ms": 11739.298272001406,
|
|
"request_id": 58,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 306.8537889647587
|
|
},
|
|
{
|
|
"e2e_ms": 11724.456365764809,
|
|
"request_id": 59,
|
|
"tpot_ms": 90.01924789792636,
|
|
"ttft_ms": 292.01188272816125
|
|
},
|
|
{
|
|
"e2e_ms": 11707.241368517316,
|
|
"request_id": 60,
|
|
"tpot_ms": 90.00056214193505,
|
|
"ttft_ms": 277.16997649156383
|
|
},
|
|
{
|
|
"e2e_ms": 11780.30673167916,
|
|
"request_id": 61,
|
|
"tpot_ms": 89.99959080602287,
|
|
"ttft_ms": 350.3586993142562
|
|
},
|
|
{
|
|
"e2e_ms": 11765.464825442563,
|
|
"request_id": 62,
|
|
"tpot_ms": 90.01924789792635,
|
|
"ttft_ms": 333.0203424059164
|
|
},
|
|
{
|
|
"e2e_ms": 11530.943387303068,
|
|
"request_id": 63,
|
|
"tpot_ms": 88.28948780420275,
|
|
"ttft_ms": 318.17843616931896
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "37e166453683700c9df7abffa278431119205a29c83e92211ffa69d90e1dca0e"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 28.99924373626709,
|
|
"offered_request_rate": 0.2,
|
|
"request_metrics_sha256": "7930ffb53e440b9eefa724912c777842a9c33cefdb46d9014e1c5003223221d2",
|
|
"request_rate_per_gpu": 0.05,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 11975.67900227787,
|
|
"request_id": 0,
|
|
"tpot_ms": 92.18976799112939,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 12131.199338988026,
|
|
"request_id": 1,
|
|
"tpot_ms": 93.2890801728389,
|
|
"ttft_ms": 283.4861570374869
|
|
},
|
|
{
|
|
"e2e_ms": 12196.411822684866,
|
|
"request_id": 2,
|
|
"tpot_ms": 93.31228634935276,
|
|
"ttft_ms": 345.75145631706584
|
|
},
|
|
{
|
|
"e2e_ms": 12171.220586311463,
|
|
"request_id": 3,
|
|
"tpot_ms": 93.31547315047263,
|
|
"ttft_ms": 320.15549620144054
|
|
},
|
|
{
|
|
"e2e_ms": 12145.933482881133,
|
|
"request_id": 4,
|
|
"tpot_ms": 93.31640326087744,
|
|
"ttft_ms": 294.75026874969853
|
|
},
|
|
{
|
|
"e2e_ms": 12120.646379450847,
|
|
"request_id": 5,
|
|
"tpot_ms": 93.31564840216181,
|
|
"ttft_ms": 269.5590323762964
|
|
},
|
|
{
|
|
"e2e_ms": 12185.858863147729,
|
|
"request_id": 6,
|
|
"tpot_ms": 93.31471829175732,
|
|
"ttft_ms": 334.88964009454844
|
|
},
|
|
{
|
|
"e2e_ms": 12160.571759717443,
|
|
"request_id": 7,
|
|
"tpot_ms": 93.31471829175764,
|
|
"ttft_ms": 309.6025366642223
|
|
},
|
|
{
|
|
"e2e_ms": 12135.284656287155,
|
|
"request_id": 8,
|
|
"tpot_ms": 93.31564840216245,
|
|
"ttft_ms": 284.19730921252295
|
|
},
|
|
{
|
|
"e2e_ms": 12200.497139984038,
|
|
"request_id": 9,
|
|
"tpot_ms": 93.31471829175764,
|
|
"ttft_ms": 349.52791693081764
|
|
},
|
|
{
|
|
"e2e_ms": 12175.305903610677,
|
|
"request_id": 10,
|
|
"tpot_ms": 93.3154731504736,
|
|
"ttft_ms": 324.2408135005306
|
|
},
|
|
{
|
|
"e2e_ms": 12150.018800180305,
|
|
"request_id": 11,
|
|
"tpot_ms": 93.31640326087775,
|
|
"ttft_ms": 298.8355860488312
|
|
},
|
|
{
|
|
"e2e_ms": 12124.731696749934,
|
|
"request_id": 12,
|
|
"tpot_ms": 93.31564840216113,
|
|
"ttft_ms": 273.6443496754717
|
|
},
|
|
{
|
|
"e2e_ms": 12189.94418044673,
|
|
"request_id": 13,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 338.97495739375927
|
|
},
|
|
{
|
|
"e2e_ms": 12164.657077016358,
|
|
"request_id": 14,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 313.68785396338694
|
|
},
|
|
{
|
|
"e2e_ms": 12139.369973585985,
|
|
"request_id": 15,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 288.2826265116023
|
|
},
|
|
{
|
|
"e2e_ms": 12204.582457282782,
|
|
"request_id": 16,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 353.6132342298117
|
|
},
|
|
{
|
|
"e2e_ms": 12179.391220909338,
|
|
"request_id": 17,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 328.3261307994394
|
|
},
|
|
{
|
|
"e2e_ms": 12154.104117478966,
|
|
"request_id": 18,
|
|
"tpot_ms": 93.31640326087647,
|
|
"ttft_ms": 302.92090334765476
|
|
},
|
|
{
|
|
"e2e_ms": 12128.817014048593,
|
|
"request_id": 19,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 277.72966697421
|
|
},
|
|
{
|
|
"e2e_ms": 12194.02949774539,
|
|
"request_id": 20,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 343.0602746924194
|
|
},
|
|
{
|
|
"e2e_ms": 12168.742394315017,
|
|
"request_id": 21,
|
|
"tpot_ms": 93.31471829175567,
|
|
"ttft_ms": 317.7731712620471
|
|
},
|
|
{
|
|
"e2e_ms": 12143.455290884645,
|
|
"request_id": 22,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 292.36794381026243
|
|
},
|
|
{
|
|
"e2e_ms": 12208.667774581443,
|
|
"request_id": 23,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 357.69855152847185
|
|
},
|
|
{
|
|
"e2e_ms": 12183.476538207999,
|
|
"request_id": 24,
|
|
"tpot_ms": 93.31547315047165,
|
|
"ttft_ms": 332.4114480980995
|
|
},
|
|
{
|
|
"e2e_ms": 12158.189434777625,
|
|
"request_id": 25,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 307.0062206463149
|
|
},
|
|
{
|
|
"e2e_ms": 12132.902331347253,
|
|
"request_id": 26,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 281.8149842728701
|
|
},
|
|
{
|
|
"e2e_ms": 12198.11481504405,
|
|
"request_id": 27,
|
|
"tpot_ms": 93.31471829175567,
|
|
"ttft_ms": 347.1455919910795
|
|
},
|
|
{
|
|
"e2e_ms": 12172.923578670605,
|
|
"request_id": 28,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 321.8584885607072
|
|
},
|
|
{
|
|
"e2e_ms": 12147.636475240233,
|
|
"request_id": 29,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 296.45326110892256
|
|
},
|
|
{
|
|
"e2e_ms": 12122.349371809862,
|
|
"request_id": 30,
|
|
"tpot_ms": 93.3156484021605,
|
|
"ttft_ms": 271.2620247354778
|
|
},
|
|
{
|
|
"e2e_ms": 12187.561855506658,
|
|
"request_id": 31,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 336.5926324536872
|
|
},
|
|
{
|
|
"e2e_ms": 12162.274752076286,
|
|
"request_id": 32,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 311.3055290233149
|
|
},
|
|
{
|
|
"e2e_ms": 12136.987648645914,
|
|
"request_id": 33,
|
|
"tpot_ms": 93.3156484021605,
|
|
"ttft_ms": 285.90030157153024
|
|
},
|
|
{
|
|
"e2e_ms": 12202.20013234271,
|
|
"request_id": 34,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 351.23090928973966
|
|
},
|
|
{
|
|
"e2e_ms": 12177.008895969266,
|
|
"request_id": 35,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 325.94380585936733
|
|
},
|
|
{
|
|
"e2e_ms": 12151.721792538894,
|
|
"request_id": 36,
|
|
"tpot_ms": 93.31640326087647,
|
|
"ttft_ms": 300.5385784075827
|
|
},
|
|
{
|
|
"e2e_ms": 12126.43468910852,
|
|
"request_id": 37,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 275.3473420341379
|
|
},
|
|
{
|
|
"e2e_ms": 12191.647172805318,
|
|
"request_id": 38,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 340.67794975234733
|
|
},
|
|
{
|
|
"e2e_ms": 12166.360069374947,
|
|
"request_id": 39,
|
|
"tpot_ms": 93.3147182917557,
|
|
"ttft_ms": 315.390846321975
|
|
},
|
|
{
|
|
"e2e_ms": 12141.072965944573,
|
|
"request_id": 40,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 289.98561887019036
|
|
},
|
|
{
|
|
"e2e_ms": 12206.285449641371,
|
|
"request_id": 41,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 355.3162265883998
|
|
},
|
|
{
|
|
"e2e_ms": 12181.094213267927,
|
|
"request_id": 42,
|
|
"tpot_ms": 93.31547315047165,
|
|
"ttft_ms": 330.02912315802746
|
|
},
|
|
{
|
|
"e2e_ms": 12155.807109837553,
|
|
"request_id": 43,
|
|
"tpot_ms": 93.31640326087646,
|
|
"ttft_ms": 304.6238957062428
|
|
},
|
|
{
|
|
"e2e_ms": 12130.520006407181,
|
|
"request_id": 44,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 279.43265933279804
|
|
},
|
|
{
|
|
"e2e_ms": 12195.73249010398,
|
|
"request_id": 45,
|
|
"tpot_ms": 93.3147182917557,
|
|
"ttft_ms": 344.76326705100746
|
|
},
|
|
{
|
|
"e2e_ms": 12170.445386673606,
|
|
"request_id": 46,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 319.47616362063513
|
|
},
|
|
{
|
|
"e2e_ms": 12145.158283243234,
|
|
"request_id": 47,
|
|
"tpot_ms": 93.31564840216049,
|
|
"ttft_ms": 294.0709361688505
|
|
},
|
|
{
|
|
"e2e_ms": 12210.37076694003,
|
|
"request_id": 48,
|
|
"tpot_ms": 93.31471829175568,
|
|
"ttft_ms": 359.4015438870599
|
|
},
|
|
{
|
|
"e2e_ms": 12185.179530566586,
|
|
"request_id": 49,
|
|
"tpot_ms": 93.31547315047163,
|
|
"ttft_ms": 334.1144404566876
|
|
},
|
|
{
|
|
"e2e_ms": 12159.892427136241,
|
|
"request_id": 50,
|
|
"tpot_ms": 93.31640326087668,
|
|
"ttft_ms": 308.70921300490295
|
|
},
|
|
{
|
|
"e2e_ms": 12134.605323705899,
|
|
"request_id": 51,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 283.51797663145817
|
|
},
|
|
{
|
|
"e2e_ms": 12199.817807402724,
|
|
"request_id": 52,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 348.848584349696
|
|
},
|
|
{
|
|
"e2e_ms": 12174.626571029308,
|
|
"request_id": 53,
|
|
"tpot_ms": 93.3154731504721,
|
|
"ttft_ms": 323.5614809193521
|
|
},
|
|
{
|
|
"e2e_ms": 12149.339467598964,
|
|
"request_id": 54,
|
|
"tpot_ms": 93.31640326087691,
|
|
"ttft_ms": 298.1562534675959
|
|
},
|
|
{
|
|
"e2e_ms": 12124.05236416862,
|
|
"request_id": 55,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 272.96501709417953
|
|
},
|
|
{
|
|
"e2e_ms": 12189.264847865445,
|
|
"request_id": 56,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 338.2956248124174
|
|
},
|
|
{
|
|
"e2e_ms": 12163.977744435102,
|
|
"request_id": 57,
|
|
"tpot_ms": 93.31471829175614,
|
|
"ttft_ms": 313.00852138207347
|
|
},
|
|
{
|
|
"e2e_ms": 12138.690641004758,
|
|
"request_id": 58,
|
|
"tpot_ms": 93.31564840216095,
|
|
"ttft_ms": 287.60329393031725
|
|
},
|
|
{
|
|
"e2e_ms": 12203.903124701583,
|
|
"request_id": 59,
|
|
"tpot_ms": 93.31471829175612,
|
|
"ttft_ms": 352.9339016485551
|
|
},
|
|
{
|
|
"e2e_ms": 12178.711888328166,
|
|
"request_id": 60,
|
|
"tpot_ms": 93.31547315047209,
|
|
"ttft_ms": 327.6467982182112
|
|
},
|
|
{
|
|
"e2e_ms": 12153.424784897823,
|
|
"request_id": 61,
|
|
"tpot_ms": 93.31640326087691,
|
|
"ttft_ms": 302.24157076645497
|
|
},
|
|
{
|
|
"e2e_ms": 11949.074413391485,
|
|
"request_id": 62,
|
|
"tpot_ms": 91.90570140943657,
|
|
"ttft_ms": 277.0503343930386
|
|
},
|
|
{
|
|
"e2e_ms": 11702.728382593137,
|
|
"request_id": 63,
|
|
"tpot_ms": 89.45155464946347,
|
|
"ttft_ms": 342.38094211127645
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "e6cebfc858981b2ae4632d2c75f5e8c1f83280e7306351e15c7cd611d2664d80"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 24.38782286643982,
|
|
"offered_request_rate": 0.4,
|
|
"request_metrics_sha256": "e62567aa33ec429331fcd9f24053a0c79db61ef288f7994e6488ae0f0fa79b7e",
|
|
"request_rate_per_gpu": 0.1,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12394.3405299904,
|
|
"request_id": 0,
|
|
"tpot_ms": 95.48631545343278,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 12520.094438712134,
|
|
"request_id": 1,
|
|
"tpot_ms": 96.04733720338449,
|
|
"ttft_ms": 322.08261388230494
|
|
},
|
|
{
|
|
"e2e_ms": 12556.880443424032,
|
|
"request_id": 2,
|
|
"tpot_ms": 96.10807445129032,
|
|
"ttft_ms": 351.1549881101628
|
|
},
|
|
{
|
|
"e2e_ms": 12502.892311945501,
|
|
"request_id": 3,
|
|
"tpot_ms": 96.14618123456218,
|
|
"ttft_ms": 292.32729515610424
|
|
},
|
|
{
|
|
"e2e_ms": 12539.549613984442,
|
|
"request_id": 4,
|
|
"tpot_ms": 96.30977229626299,
|
|
"ttft_ms": 308.2085323590427
|
|
},
|
|
{
|
|
"e2e_ms": 12576.335618696337,
|
|
"request_id": 5,
|
|
"tpot_ms": 96.32399459689755,
|
|
"ttft_ms": 343.188304890349
|
|
},
|
|
{
|
|
"e2e_ms": 12522.347487217807,
|
|
"request_id": 6,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 289.200173411821
|
|
},
|
|
{
|
|
"e2e_ms": 12559.133491929704,
|
|
"request_id": 7,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 325.98617812371697
|
|
},
|
|
{
|
|
"e2e_ms": 12505.145360451173,
|
|
"request_id": 8,
|
|
"tpot_ms": 96.32399459689753,
|
|
"ttft_ms": 271.99804664518723
|
|
},
|
|
{
|
|
"e2e_ms": 12541.802662490163,
|
|
"request_id": 9,
|
|
"tpot_ms": 96.32399459689792,
|
|
"ttft_ms": 308.6553486841268
|
|
},
|
|
{
|
|
"e2e_ms": 12578.588667202104,
|
|
"request_id": 10,
|
|
"tpot_ms": 96.32399459689827,
|
|
"ttft_ms": 345.4413533960228
|
|
},
|
|
{
|
|
"e2e_ms": 12524.60053572362,
|
|
"request_id": 11,
|
|
"tpot_ms": 96.32399459689864,
|
|
"ttft_ms": 291.453221917493
|
|
},
|
|
{
|
|
"e2e_ms": 12561.386540435564,
|
|
"request_id": 12,
|
|
"tpot_ms": 96.32399459689903,
|
|
"ttft_ms": 328.239226629389
|
|
},
|
|
{
|
|
"e2e_ms": 12507.39840895708,
|
|
"request_id": 13,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 274.2510951508592
|
|
},
|
|
{
|
|
"e2e_ms": 12544.05571099607,
|
|
"request_id": 14,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 310.90839718984853
|
|
},
|
|
{
|
|
"e2e_ms": 12580.841715708011,
|
|
"request_id": 15,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 347.6944019017907
|
|
},
|
|
{
|
|
"e2e_ms": 12526.853584229528,
|
|
"request_id": 16,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 293.7062704233071
|
|
},
|
|
{
|
|
"e2e_ms": 12563.63958894147,
|
|
"request_id": 17,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 330.49227513524926
|
|
},
|
|
{
|
|
"e2e_ms": 12509.651457462987,
|
|
"request_id": 18,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 276.5041436567657
|
|
},
|
|
{
|
|
"e2e_ms": 12546.308759501975,
|
|
"request_id": 19,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 313.161445695755
|
|
},
|
|
{
|
|
"e2e_ms": 12583.094764213918,
|
|
"request_id": 20,
|
|
"tpot_ms": 96.32399459689938,
|
|
"ttft_ms": 349.94745040769715
|
|
},
|
|
{
|
|
"e2e_ms": 12529.106632735506,
|
|
"request_id": 21,
|
|
"tpot_ms": 96.32399459689994,
|
|
"ttft_ms": 295.95931892921357
|
|
},
|
|
{
|
|
"e2e_ms": 12565.892637447447,
|
|
"request_id": 22,
|
|
"tpot_ms": 96.32399459689992,
|
|
"ttft_ms": 332.7453236411557
|
|
},
|
|
{
|
|
"e2e_ms": 12511.904505968956,
|
|
"request_id": 23,
|
|
"tpot_ms": 96.32399459689988,
|
|
"ttft_ms": 278.75719216267214
|
|
},
|
|
{
|
|
"e2e_ms": 12548.561808007931,
|
|
"request_id": 24,
|
|
"tpot_ms": 96.32399459689977,
|
|
"ttft_ms": 315.41449420166146
|
|
},
|
|
{
|
|
"e2e_ms": 12585.347812719874,
|
|
"request_id": 25,
|
|
"tpot_ms": 96.32399459689977,
|
|
"ttft_ms": 352.2004989136036
|
|
},
|
|
{
|
|
"e2e_ms": 12531.359681241383,
|
|
"request_id": 26,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 298.2123674351982
|
|
},
|
|
{
|
|
"e2e_ms": 12568.145685953326,
|
|
"request_id": 27,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 334.99837214714034
|
|
},
|
|
{
|
|
"e2e_ms": 12514.157554474836,
|
|
"request_id": 28,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 281.01024066864966
|
|
},
|
|
{
|
|
"e2e_ms": 12550.81485651381,
|
|
"request_id": 29,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 317.66754270762476
|
|
},
|
|
{
|
|
"e2e_ms": 12587.600861225752,
|
|
"request_id": 30,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 354.4535474195669
|
|
},
|
|
{
|
|
"e2e_ms": 12533.612729747261,
|
|
"request_id": 31,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 300.46541594107623
|
|
},
|
|
{
|
|
"e2e_ms": 12570.398734459204,
|
|
"request_id": 32,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 337.2514206530184
|
|
},
|
|
{
|
|
"e2e_ms": 12516.410602980714,
|
|
"request_id": 33,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 283.2632891745277
|
|
},
|
|
{
|
|
"e2e_ms": 12553.067905019689,
|
|
"request_id": 34,
|
|
"tpot_ms": 96.32399459689911,
|
|
"ttft_ms": 319.9205912135028
|
|
},
|
|
{
|
|
"e2e_ms": 12589.85390973163,
|
|
"request_id": 35,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 356.70659592544496
|
|
},
|
|
{
|
|
"e2e_ms": 12535.865778253139,
|
|
"request_id": 36,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 302.7184644469543
|
|
},
|
|
{
|
|
"e2e_ms": 12572.651782965082,
|
|
"request_id": 37,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 339.5044691588964
|
|
},
|
|
{
|
|
"e2e_ms": 12518.663651486591,
|
|
"request_id": 38,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 285.51633768040574
|
|
},
|
|
{
|
|
"e2e_ms": 12555.320953525566,
|
|
"request_id": 39,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 322.17363971938084
|
|
},
|
|
{
|
|
"e2e_ms": 12770.519034100516,
|
|
"request_id": 40,
|
|
"tpot_ms": 97.72881409188341,
|
|
"ttft_ms": 358.959644431323
|
|
},
|
|
{
|
|
"e2e_ms": 12538.070601439798,
|
|
"request_id": 41,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 304.9715129528323
|
|
},
|
|
{
|
|
"e2e_ms": 12574.856606151741,
|
|
"request_id": 42,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 341.75751766477447
|
|
},
|
|
{
|
|
"e2e_ms": 12520.86847467325,
|
|
"request_id": 43,
|
|
"tpot_ms": 96.32361486997611,
|
|
"ttft_ms": 287.7693861862838
|
|
},
|
|
{
|
|
"e2e_ms": 12557.654479385192,
|
|
"request_id": 44,
|
|
"tpot_ms": 96.32462827684986,
|
|
"ttft_ms": 324.4266882252589
|
|
},
|
|
{
|
|
"e2e_ms": 12503.666347906701,
|
|
"request_id": 45,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 270.51903410051636
|
|
},
|
|
{
|
|
"e2e_ms": 12540.323649945676,
|
|
"request_id": 46,
|
|
"tpot_ms": 96.3239945968991,
|
|
"ttft_ms": 307.17633613949147
|
|
},
|
|
{
|
|
"e2e_ms": 12577.109654657448,
|
|
"request_id": 47,
|
|
"tpot_ms": 96.32399459689775,
|
|
"ttft_ms": 343.9623408514336
|
|
},
|
|
{
|
|
"e2e_ms": 12523.121523178786,
|
|
"request_id": 48,
|
|
"tpot_ms": 96.3239945968964,
|
|
"ttft_ms": 289.97420937294294
|
|
},
|
|
{
|
|
"e2e_ms": 12559.907527890544,
|
|
"request_id": 49,
|
|
"tpot_ms": 96.32399459689495,
|
|
"ttft_ms": 326.7602140848851
|
|
},
|
|
{
|
|
"e2e_ms": 12505.919396411884,
|
|
"request_id": 50,
|
|
"tpot_ms": 96.32399459689363,
|
|
"ttft_ms": 272.7720826063944
|
|
},
|
|
{
|
|
"e2e_ms": 12542.576698450688,
|
|
"request_id": 51,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 309.4293846453695
|
|
},
|
|
{
|
|
"e2e_ms": 12579.362703162446,
|
|
"request_id": 52,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 346.2153893571269
|
|
},
|
|
{
|
|
"e2e_ms": 12525.374571683784,
|
|
"request_id": 53,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 292.2272578784657
|
|
},
|
|
{
|
|
"e2e_ms": 12562.160576395541,
|
|
"request_id": 54,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 329.0132625902231
|
|
},
|
|
{
|
|
"e2e_ms": 12508.17244491688,
|
|
"request_id": 55,
|
|
"tpot_ms": 96.32399459689226,
|
|
"ttft_ms": 275.0251311115619
|
|
},
|
|
{
|
|
"e2e_ms": 12544.829746955686,
|
|
"request_id": 56,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 311.6824331503665
|
|
},
|
|
{
|
|
"e2e_ms": 12581.615751667443,
|
|
"request_id": 57,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 348.4684378621239
|
|
},
|
|
{
|
|
"e2e_ms": 12527.627620188781,
|
|
"request_id": 58,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 294.4803063834627
|
|
},
|
|
{
|
|
"e2e_ms": 12564.413624900539,
|
|
"request_id": 59,
|
|
"tpot_ms": 96.32399459689228,
|
|
"ttft_ms": 331.2663110952201
|
|
},
|
|
{
|
|
"e2e_ms": 12314.305135768365,
|
|
"request_id": 60,
|
|
"tpot_ms": 94.77973981221895,
|
|
"ttft_ms": 277.2781796165589
|
|
},
|
|
{
|
|
"e2e_ms": 12170.36562563149,
|
|
"request_id": 61,
|
|
"tpot_ms": 93.35771766910335,
|
|
"ttft_ms": 313.93548165536345
|
|
},
|
|
{
|
|
"e2e_ms": 12023.933572014499,
|
|
"request_id": 62,
|
|
"tpot_ms": 91.91505579249903,
|
|
"ttft_ms": 350.72148636712086
|
|
},
|
|
{
|
|
"e2e_ms": 11724.699298497455,
|
|
"request_id": 63,
|
|
"tpot_ms": 89.98398380794484,
|
|
"ttft_ms": 296.73335488845964
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "021442528b6c4067d63827496b66b6f530adbe9f4b29732b1aa7aab2112ced9f"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 22.886351823806763,
|
|
"offered_request_rate": 0.8,
|
|
"request_metrics_sha256": "b39b2c6a1c6236e30b246303e0b4d66b80a2515899d8c815efc015ef3f70db54",
|
|
"request_rate_per_gpu": 0.2,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 14028.106647728775,
|
|
"request_id": 0,
|
|
"tpot_ms": 108.35061559310502,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 14267.352384844213,
|
|
"request_id": 1,
|
|
"tpot_ms": 109.65331923259451,
|
|
"ttft_ms": 341.3808423047111
|
|
},
|
|
{
|
|
"e2e_ms": 14406.16288598222,
|
|
"request_id": 2,
|
|
"tpot_ms": 110.64064370419715,
|
|
"ttft_ms": 354.8011355491818
|
|
},
|
|
{
|
|
"e2e_ms": 14444.508775243683,
|
|
"request_id": 3,
|
|
"tpot_ms": 111.53949237897281,
|
|
"ttft_ms": 278.99324311413756
|
|
},
|
|
{
|
|
"e2e_ms": 14585.312621821135,
|
|
"request_id": 4,
|
|
"tpot_ms": 112.58765687327464,
|
|
"ttft_ms": 286.6801989152554
|
|
},
|
|
{
|
|
"e2e_ms": 14727.157204966976,
|
|
"request_id": 5,
|
|
"tpot_ms": 113.57630070139629,
|
|
"ttft_ms": 302.9670158896467
|
|
},
|
|
{
|
|
"e2e_ms": 14871.045643448613,
|
|
"request_id": 6,
|
|
"tpot_ms": 114.43037546464966,
|
|
"ttft_ms": 338.38795943810675
|
|
},
|
|
{
|
|
"e2e_ms": 14913.487485283544,
|
|
"request_id": 7,
|
|
"tpot_ms": 115.12950641852738,
|
|
"ttft_ms": 292.0401701305657
|
|
},
|
|
{
|
|
"e2e_ms": 15059.457396901958,
|
|
"request_id": 8,
|
|
"tpot_ms": 115.80285438725795,
|
|
"ttft_ms": 352.4948897201998
|
|
},
|
|
{
|
|
"e2e_ms": 15271.481983406971,
|
|
"request_id": 9,
|
|
"tpot_ms": 117.58757541921905,
|
|
"ttft_ms": 337.85990516615175
|
|
},
|
|
{
|
|
"e2e_ms": 15316.138289012584,
|
|
"request_id": 10,
|
|
"tpot_ms": 117.88145073243837,
|
|
"ttft_ms": 345.1940459929119
|
|
},
|
|
{
|
|
"e2e_ms": 15090.835365768904,
|
|
"request_id": 11,
|
|
"tpot_ms": 116.63565919716638,
|
|
"ttft_ms": 278.1066477287748
|
|
},
|
|
{
|
|
"e2e_ms": 15302.859952273917,
|
|
"request_id": 12,
|
|
"tpot_ms": 118.0348305674148,
|
|
"ttft_ms": 312.43647021223705
|
|
},
|
|
{
|
|
"e2e_ms": 15347.54563377864,
|
|
"request_id": 13,
|
|
"tpot_ms": 118.10044582475892,
|
|
"ttft_ms": 348.7890140342564
|
|
},
|
|
{
|
|
"e2e_ms": 15289.743982068245,
|
|
"request_id": 14,
|
|
"tpot_ms": 118.15013414242924,
|
|
"ttft_ms": 284.6769459797329
|
|
},
|
|
{
|
|
"e2e_ms": 15334.400287673823,
|
|
"request_id": 15,
|
|
"tpot_ms": 118.19982246009941,
|
|
"ttft_ms": 323.022835241197
|
|
},
|
|
{
|
|
"e2e_ms": 15380.492763872971,
|
|
"request_id": 16,
|
|
"tpot_ms": 118.2528557378034,
|
|
"ttft_ms": 362.38008517193873
|
|
},
|
|
{
|
|
"e2e_ms": 15322.691112162565,
|
|
"request_id": 17,
|
|
"tpot_ms": 118.27848723653601,
|
|
"ttft_ms": 301.32323312249196
|
|
},
|
|
{
|
|
"e2e_ms": 15367.376793667254,
|
|
"request_id": 18,
|
|
"tpot_ms": 118.29615526543176,
|
|
"ttft_ms": 343.765074957421
|
|
},
|
|
{
|
|
"e2e_ms": 15309.575141956848,
|
|
"request_id": 19,
|
|
"tpot_ms": 118.30562851819849,
|
|
"ttft_ms": 284.7603201456401
|
|
},
|
|
{
|
|
"e2e_ms": 15354.260823461536,
|
|
"request_id": 20,
|
|
"tpot_ms": 118.30690699483614,
|
|
"ttft_ms": 329.2836351173456
|
|
},
|
|
{
|
|
"e2e_ms": 15296.45917175113,
|
|
"request_id": 21,
|
|
"tpot_ms": 118.3069069948359,
|
|
"ttft_ms": 271.4819834069715
|
|
},
|
|
{
|
|
"e2e_ms": 15341.115477356709,
|
|
"request_id": 22,
|
|
"tpot_ms": 118.30690699483563,
|
|
"ttft_ms": 316.138289012585
|
|
},
|
|
{
|
|
"e2e_ms": 15387.207953555859,
|
|
"request_id": 23,
|
|
"tpot_ms": 118.31926259505173,
|
|
"ttft_ms": 360.6616039842905
|
|
},
|
|
{
|
|
"e2e_ms": 15329.406301845453,
|
|
"request_id": 24,
|
|
"tpot_ms": 118.31926259505147,
|
|
"ttft_ms": 302.8599522739164
|
|
},
|
|
{
|
|
"e2e_ms": 15375.4987780446,
|
|
"request_id": 25,
|
|
"tpot_ms": 118.33033971862962,
|
|
"ttft_ms": 347.5456337786404
|
|
},
|
|
{
|
|
"e2e_ms": 15317.697126334195,
|
|
"request_id": 26,
|
|
"tpot_ms": 118.33033971862953,
|
|
"ttft_ms": 289.74398206824503
|
|
},
|
|
{
|
|
"e2e_ms": 15362.382807838883,
|
|
"request_id": 27,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 334.400287673823
|
|
},
|
|
{
|
|
"e2e_ms": 15304.581156128477,
|
|
"request_id": 28,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 276.59863596341694
|
|
},
|
|
{
|
|
"e2e_ms": 15349.237461734056,
|
|
"request_id": 29,
|
|
"tpot_ms": 118.3192625950511,
|
|
"ttft_ms": 322.6911121625662
|
|
},
|
|
{
|
|
"e2e_ms": 15395.329937933206,
|
|
"request_id": 30,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 367.37679366725473
|
|
},
|
|
{
|
|
"e2e_ms": 15337.5282862228,
|
|
"request_id": 31,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 309.57514195684865
|
|
},
|
|
{
|
|
"e2e_ms": 15383.620762421948,
|
|
"request_id": 32,
|
|
"tpot_ms": 118.34141684220796,
|
|
"ttft_ms": 354.2608234615372
|
|
},
|
|
{
|
|
"e2e_ms": 15325.819110711542,
|
|
"request_id": 33,
|
|
"tpot_ms": 118.34141684220796,
|
|
"ttft_ms": 296.4591717511311
|
|
},
|
|
{
|
|
"e2e_ms": 15370.50479221623,
|
|
"request_id": 34,
|
|
"tpot_ms": 118.34164814850017,
|
|
"ttft_ms": 341.1154773567091
|
|
},
|
|
{
|
|
"e2e_ms": 15312.703140505824,
|
|
"request_id": 35,
|
|
"tpot_ms": 118.34164814850017,
|
|
"ttft_ms": 283.31382564630303
|
|
},
|
|
{
|
|
"e2e_ms": 15357.388822010513,
|
|
"request_id": 36,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 329.4063018454523
|
|
},
|
|
{
|
|
"e2e_ms": 15299.587170300107,
|
|
"request_id": 37,
|
|
"tpot_ms": 118.33057102492174,
|
|
"ttft_ms": 271.6046501350462
|
|
},
|
|
{
|
|
"e2e_ms": 15344.243475905685,
|
|
"request_id": 38,
|
|
"tpot_ms": 118.3192625950511,
|
|
"ttft_ms": 317.69712633419545
|
|
},
|
|
{
|
|
"e2e_ms": 15390.335952104835,
|
|
"request_id": 39,
|
|
"tpot_ms": 118.33033971862955,
|
|
"ttft_ms": 362.382807838884
|
|
},
|
|
{
|
|
"e2e_ms": 15332.53430039447,
|
|
"request_id": 40,
|
|
"tpot_ms": 118.33033971862987,
|
|
"ttft_ms": 304.5811561284779
|
|
},
|
|
{
|
|
"e2e_ms": 15378.626776593663,
|
|
"request_id": 41,
|
|
"tpot_ms": 118.34164814850084,
|
|
"ttft_ms": 349.2374617340559
|
|
},
|
|
{
|
|
"e2e_ms": 15320.8251248833,
|
|
"request_id": 42,
|
|
"tpot_ms": 118.34164814850119,
|
|
"ttft_ms": 291.4358100236498
|
|
},
|
|
{
|
|
"e2e_ms": 15365.51080638803,
|
|
"request_id": 43,
|
|
"tpot_ms": 118.33057102492307,
|
|
"ttft_ms": 337.52828622279907
|
|
},
|
|
{
|
|
"e2e_ms": 15307.709154677666,
|
|
"request_id": 44,
|
|
"tpot_ms": 118.33057102492342,
|
|
"ttft_ms": 279.726634512393
|
|
},
|
|
{
|
|
"e2e_ms": 15352.365460283294,
|
|
"request_id": 45,
|
|
"tpot_ms": 118.31926259505316,
|
|
"ttft_ms": 325.81911071154224
|
|
},
|
|
{
|
|
"e2e_ms": 15398.457936482486,
|
|
"request_id": 46,
|
|
"tpot_ms": 118.33033971863193,
|
|
"ttft_ms": 370.5047922162308
|
|
},
|
|
{
|
|
"e2e_ms": 15340.656284772123,
|
|
"request_id": 47,
|
|
"tpot_ms": 118.33033971863227,
|
|
"ttft_ms": 312.7031405058247
|
|
},
|
|
{
|
|
"e2e_ms": 15386.748760971315,
|
|
"request_id": 48,
|
|
"tpot_ms": 118.34141684221105,
|
|
"ttft_ms": 357.38882201051325
|
|
},
|
|
{
|
|
"e2e_ms": 15328.947109260951,
|
|
"request_id": 49,
|
|
"tpot_ms": 118.34141684221137,
|
|
"ttft_ms": 299.5871703001072
|
|
},
|
|
{
|
|
"e2e_ms": 15375.039585460143,
|
|
"request_id": 50,
|
|
"tpot_ms": 118.35272527208234,
|
|
"ttft_ms": 344.24347590568516
|
|
},
|
|
{
|
|
"e2e_ms": 15317.23793374978,
|
|
"request_id": 51,
|
|
"tpot_ms": 118.3527252720827,
|
|
"ttft_ms": 286.4418241952791
|
|
},
|
|
{
|
|
"e2e_ms": 15194.4223437212,
|
|
"request_id": 52,
|
|
"tpot_ms": 117.02274049863567,
|
|
"ttft_ms": 332.53430039447096
|
|
},
|
|
{
|
|
"e2e_ms": 14959.152693280232,
|
|
"request_id": 53,
|
|
"tpot_ms": 115.62535468185925,
|
|
"ttft_ms": 274.7326486841075
|
|
},
|
|
{
|
|
"e2e_ms": 14796.032219342294,
|
|
"request_id": 54,
|
|
"tpot_ms": 113.97800861778735,
|
|
"ttft_ms": 320.8251248832994
|
|
},
|
|
{
|
|
"e2e_ms": 14608.764053703837,
|
|
"request_id": 55,
|
|
"tpot_ms": 112.1516003725654,
|
|
"ttft_ms": 365.5108063880306
|
|
},
|
|
{
|
|
"e2e_ms": 14305.495223453576,
|
|
"request_id": 56,
|
|
"tpot_ms": 110.21878794311739,
|
|
"ttft_ms": 307.70915467766713
|
|
},
|
|
{
|
|
"e2e_ms": 14083.450969580696,
|
|
"request_id": 57,
|
|
"tpot_ms": 108.11878353777482,
|
|
"ttft_ms": 352.36546028329485
|
|
},
|
|
{
|
|
"e2e_ms": 13758.528543574443,
|
|
"request_id": 58,
|
|
"tpot_ms": 106.01547035434261,
|
|
"ttft_ms": 294.5638085729314
|
|
},
|
|
{
|
|
"e2e_ms": 13507.044041669204,
|
|
"request_id": 59,
|
|
"tpot_ms": 103.67234454249669,
|
|
"ttft_ms": 340.6562847721233
|
|
},
|
|
{
|
|
"e2e_ms": 13157.052579489218,
|
|
"request_id": 60,
|
|
"tpot_ms": 101.37163737344456,
|
|
"ttft_ms": 282.85463306175984
|
|
},
|
|
{
|
|
"e2e_ms": 12903.847402123616,
|
|
"request_id": 61,
|
|
"tpot_ms": 99.01496293592649,
|
|
"ttft_ms": 328.9471092609517
|
|
},
|
|
{
|
|
"e2e_ms": 12559.065843040158,
|
|
"request_id": 62,
|
|
"tpot_ms": 96.75527862590212,
|
|
"ttft_ms": 271.1454575505883
|
|
},
|
|
{
|
|
"e2e_ms": 12277.402762692504,
|
|
"request_id": 63,
|
|
"tpot_ms": 94.17452621214743,
|
|
"ttft_ms": 317.23793374978015
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "9ed2bf69d3193253248833822a357db5130e6ca29282e93e04cc92961dba468d"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 27.091737508773804,
|
|
"offered_request_rate": 1.2,
|
|
"request_metrics_sha256": "3825f14be2103486ce094e0b335c86deef683645cae8f05ef3d4beb6fb24cf95",
|
|
"request_rate_per_gpu": 0.3,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 15728.871662360909,
|
|
"request_id": 0,
|
|
"tpot_ms": 121.74246610201946,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 15904.548256233129,
|
|
"request_id": 1,
|
|
"tpot_ms": 122.72470152955445,
|
|
"ttft_ms": 318.5111619797136
|
|
},
|
|
{
|
|
"e2e_ms": 16140.50586206045,
|
|
"request_id": 2,
|
|
"tpot_ms": 124.75532551591714,
|
|
"ttft_ms": 296.5795215389744
|
|
},
|
|
{
|
|
"e2e_ms": 16211.825024161684,
|
|
"request_id": 3,
|
|
"tpot_ms": 125.48331552046938,
|
|
"ttft_ms": 275.44395306207383
|
|
},
|
|
{
|
|
"e2e_ms": 16388.84099440166,
|
|
"request_id": 4,
|
|
"tpot_ms": 126.36635826198417,
|
|
"ttft_ms": 340.31349512967114
|
|
},
|
|
{
|
|
"e2e_ms": 16460.2590055204,
|
|
"request_id": 5,
|
|
"tpot_ms": 127.08834011289514,
|
|
"ttft_ms": 320.03981118271605
|
|
},
|
|
{
|
|
"e2e_ms": 16531.726441149392,
|
|
"request_id": 6,
|
|
"tpot_ms": 127.72809453706097,
|
|
"ttft_ms": 310.25843494264916
|
|
},
|
|
{
|
|
"e2e_ms": 16603.193876778383,
|
|
"request_id": 7,
|
|
"tpot_ms": 128.3170301134443,
|
|
"ttft_ms": 306.93105237095784
|
|
},
|
|
{
|
|
"e2e_ms": 16674.731660081485,
|
|
"request_id": 8,
|
|
"tpot_ms": 128.84921275092722,
|
|
"ttft_ms": 310.8816407137276
|
|
},
|
|
{
|
|
"e2e_ms": 16746.318867894835,
|
|
"request_id": 9,
|
|
"tpot_ms": 129.28898520065795,
|
|
"ttft_ms": 326.61774741127834
|
|
},
|
|
{
|
|
"e2e_ms": 16982.978169021266,
|
|
"request_id": 10,
|
|
"tpot_ms": 130.92468009104587,
|
|
"ttft_ms": 355.5437974584379
|
|
},
|
|
{
|
|
"e2e_ms": 16948.91870099869,
|
|
"request_id": 11,
|
|
"tpot_ms": 131.099598546656,
|
|
"ttft_ms": 299.26968557337295
|
|
},
|
|
{
|
|
"e2e_ms": 17020.55604111586,
|
|
"request_id": 12,
|
|
"tpot_ms": 131.2589874619586,
|
|
"ttft_ms": 350.6646334471153
|
|
},
|
|
{
|
|
"e2e_ms": 16986.496573094286,
|
|
"request_id": 13,
|
|
"tpot_ms": 131.335760528223,
|
|
"ttft_ms": 306.854986009963
|
|
},
|
|
{
|
|
"e2e_ms": 17059.03976122884,
|
|
"request_id": 14,
|
|
"tpot_ms": 131.4398579526774,
|
|
"ttft_ms": 366.1778012388126
|
|
},
|
|
{
|
|
"e2e_ms": 17024.98029320726,
|
|
"request_id": 15,
|
|
"tpot_ms": 131.50690676411472,
|
|
"ttft_ms": 323.6031341646921
|
|
},
|
|
{
|
|
"e2e_ms": 16990.920825185683,
|
|
"request_id": 16,
|
|
"tpot_ms": 131.56847300071718,
|
|
"ttft_ms": 281.7247540946006
|
|
},
|
|
{
|
|
"e2e_ms": 17063.464013320234,
|
|
"request_id": 17,
|
|
"tpot_ms": 131.63886355765484,
|
|
"ttft_ms": 345.32834149806746
|
|
},
|
|
{
|
|
"e2e_ms": 17029.40454529866,
|
|
"request_id": 18,
|
|
"tpot_ms": 131.67812927130495,
|
|
"ttft_ms": 306.28212784293396
|
|
},
|
|
{
|
|
"e2e_ms": 16995.3450772771,
|
|
"request_id": 19,
|
|
"tpot_ms": 131.6904025418972,
|
|
"ttft_ms": 270.66395445615575
|
|
},
|
|
{
|
|
"e2e_ms": 17067.888265411675,
|
|
"request_id": 20,
|
|
"tpot_ms": 131.7034495326512,
|
|
"ttft_ms": 341.5501747649721
|
|
},
|
|
{
|
|
"e2e_ms": 17033.828797390113,
|
|
"request_id": 21,
|
|
"tpot_ms": 131.70595487135952,
|
|
"ttft_ms": 307.17252872745246
|
|
},
|
|
{
|
|
"e2e_ms": 16999.769329368555,
|
|
"request_id": 22,
|
|
"tpot_ms": 131.7084602100679,
|
|
"ttft_ms": 272.79488268993293
|
|
},
|
|
{
|
|
"e2e_ms": 17072.312517503127,
|
|
"request_id": 23,
|
|
"tpot_ms": 131.71809821033827,
|
|
"ttft_ms": 344.1140447901674
|
|
},
|
|
{
|
|
"e2e_ms": 17038.25304948157,
|
|
"request_id": 24,
|
|
"tpot_ms": 131.71982521031822,
|
|
"ttft_ms": 309.83524777115434
|
|
},
|
|
{
|
|
"e2e_ms": 17004.193581460007,
|
|
"request_id": 25,
|
|
"tpot_ms": 131.72116304093396,
|
|
"ttft_ms": 275.60587526139454
|
|
},
|
|
{
|
|
"e2e_ms": 17076.736769594583,
|
|
"request_id": 26,
|
|
"tpot_ms": 131.72963353311178,
|
|
"ttft_ms": 347.07331088938884
|
|
},
|
|
{
|
|
"e2e_ms": 17042.67730157302,
|
|
"request_id": 27,
|
|
"tpot_ms": 131.7305821943633,
|
|
"ttft_ms": 312.8933628888824
|
|
},
|
|
{
|
|
"e2e_ms": 17008.617833551463,
|
|
"request_id": 28,
|
|
"tpot_ms": 131.7309769369132,
|
|
"ttft_ms": 278.783762563485
|
|
},
|
|
{
|
|
"e2e_ms": 17081.161021686035,
|
|
"request_id": 29,
|
|
"tpot_ms": 131.73850434102513,
|
|
"ttft_ms": 350.37097037584175
|
|
},
|
|
{
|
|
"e2e_ms": 17047.986478516996,
|
|
"request_id": 30,
|
|
"tpot_ms": 131.74547225324986,
|
|
"ttft_ms": 316.3115023542638
|
|
},
|
|
{
|
|
"e2e_ms": 17013.927010495438,
|
|
"request_id": 31,
|
|
"tpot_ms": 131.74547225325,
|
|
"ttft_ms": 282.25203433268575
|
|
},
|
|
{
|
|
"e2e_ms": 17086.47019863001,
|
|
"request_id": 32,
|
|
"tpot_ms": 131.75260491481217,
|
|
"ttft_ms": 353.8893744488618
|
|
},
|
|
{
|
|
"e2e_ms": 17053.29565546097,
|
|
"request_id": 33,
|
|
"tpot_ms": 131.7595728270369,
|
|
"ttft_ms": 319.82990642728385
|
|
},
|
|
{
|
|
"e2e_ms": 17019.236187439412,
|
|
"request_id": 34,
|
|
"tpot_ms": 131.75957282703703,
|
|
"ttft_ms": 285.7704384057058
|
|
},
|
|
{
|
|
"e2e_ms": 17091.779375573984,
|
|
"request_id": 35,
|
|
"tpot_ms": 131.7595728270372,
|
|
"ttft_ms": 358.31362654026225
|
|
},
|
|
{
|
|
"e2e_ms": 17058.604832404944,
|
|
"request_id": 36,
|
|
"tpot_ms": 131.7665407392619,
|
|
"ttft_ms": 324.2541585186842
|
|
},
|
|
{
|
|
"e2e_ms": 17024.545364383386,
|
|
"request_id": 37,
|
|
"tpot_ms": 131.76654073926207,
|
|
"ttft_ms": 290.1946904971062
|
|
},
|
|
{
|
|
"e2e_ms": 17097.08855251796,
|
|
"request_id": 38,
|
|
"tpot_ms": 131.76654073926218,
|
|
"ttft_ms": 362.7378786316626
|
|
},
|
|
{
|
|
"e2e_ms": 17063.934932514778,
|
|
"request_id": 39,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 328.6784106101024
|
|
},
|
|
{
|
|
"e2e_ms": 17029.875464493216,
|
|
"request_id": 40,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 294.61894258854215
|
|
},
|
|
{
|
|
"e2e_ms": 17102.41865262779,
|
|
"request_id": 41,
|
|
"tpot_ms": 131.7736734008242,
|
|
"ttft_ms": 367.1621307231163
|
|
},
|
|
{
|
|
"e2e_ms": 17069.265032624608,
|
|
"request_id": 42,
|
|
"tpot_ms": 131.78080606238623,
|
|
"ttft_ms": 333.1026627015561
|
|
},
|
|
{
|
|
"e2e_ms": 17035.205564603046,
|
|
"request_id": 43,
|
|
"tpot_ms": 131.78080606238623,
|
|
"ttft_ms": 299.04319467999585
|
|
},
|
|
{
|
|
"e2e_ms": 16941.812657737883,
|
|
"request_id": 44,
|
|
"tpot_ms": 130.47422263719145,
|
|
"ttft_ms": 371.58638281457
|
|
},
|
|
{
|
|
"e2e_ms": 16742.48916517699,
|
|
"request_id": 45,
|
|
"tpot_ms": 129.1729311053857,
|
|
"ttft_ms": 337.5269147930098
|
|
},
|
|
{
|
|
"e2e_ms": 16538.03643831958,
|
|
"request_id": 46,
|
|
"tpot_ms": 127.83125190195379,
|
|
"ttft_ms": 303.46744677144954
|
|
},
|
|
{
|
|
"e2e_ms": 16435.957848641778,
|
|
"request_id": 47,
|
|
"tpot_ms": 126.45627727350987,
|
|
"ttft_ms": 376.0106349060237
|
|
},
|
|
{
|
|
"e2e_ms": 16228.476901947985,
|
|
"request_id": 48,
|
|
"tpot_ms": 125.09075381939779,
|
|
"ttft_ms": 341.9511668844635
|
|
},
|
|
{
|
|
"e2e_ms": 16020.052127617348,
|
|
"request_id": 49,
|
|
"tpot_ms": 123.7177986516098,
|
|
"ttft_ms": 307.89169886290324
|
|
},
|
|
{
|
|
"e2e_ms": 15809.427558622445,
|
|
"request_id": 50,
|
|
"tpot_ms": 122.32055435377619,
|
|
"ttft_ms": 274.71715569286914
|
|
},
|
|
{
|
|
"e2e_ms": 15703.511289947706,
|
|
"request_id": 51,
|
|
"tpot_ms": 120.91536178046667,
|
|
"ttft_ms": 347.26034382843807
|
|
},
|
|
{
|
|
"e2e_ms": 15485.101955905271,
|
|
"request_id": 52,
|
|
"tpot_ms": 119.46378803227081,
|
|
"ttft_ms": 313.20087580687783
|
|
},
|
|
{
|
|
"e2e_ms": 15260.624457846197,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.95746555282956,
|
|
"ttft_ms": 280.02633263684373
|
|
},
|
|
{
|
|
"e2e_ms": 15118.941732007286,
|
|
"request_id": 54,
|
|
"tpot_ms": 116.27064733255806,
|
|
"ttft_ms": 352.56952077241266
|
|
},
|
|
{
|
|
"e2e_ms": 14865.280308326035,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.54149807539514,
|
|
"ttft_ms": 318.5100527508524
|
|
},
|
|
{
|
|
"e2e_ms": 14599.985676841925,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.71378084457564,
|
|
"ttft_ms": 285.3355095808183
|
|
},
|
|
{
|
|
"e2e_ms": 14420.806000135271,
|
|
"request_id": 57,
|
|
"tpot_ms": 110.73171104266838,
|
|
"ttft_ms": 357.87869771638725
|
|
},
|
|
{
|
|
"e2e_ms": 14142.519211198518,
|
|
"request_id": 58,
|
|
"tpot_ms": 108.80866127168261,
|
|
"ttft_ms": 323.819229694827
|
|
},
|
|
{
|
|
"e2e_ms": 13853.830695007076,
|
|
"request_id": 59,
|
|
"tpot_ms": 106.79657547493251,
|
|
"ttft_ms": 290.6656096906488
|
|
},
|
|
{
|
|
"e2e_ms": 13650.503338148035,
|
|
"request_id": 60,
|
|
"tpot_ms": 104.62436645922692,
|
|
"ttft_ms": 363.20879782621773
|
|
},
|
|
{
|
|
"e2e_ms": 13360.876271706531,
|
|
"request_id": 61,
|
|
"tpot_ms": 102.61202316458169,
|
|
"ttft_ms": 329.1493298046575
|
|
},
|
|
{
|
|
"e2e_ms": 13070.674002922458,
|
|
"request_id": 62,
|
|
"tpot_ms": 100.58801805607857,
|
|
"ttft_ms": 295.9957098004793
|
|
},
|
|
{
|
|
"e2e_ms": 12853.555073004585,
|
|
"request_id": 63,
|
|
"tpot_ms": 98.30721397691761,
|
|
"ttft_ms": 368.5388979360482
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.1875,
|
|
"passed": 12
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "3e07b28ac20428d11b8d83064d68561f03804f8adf37111de43398fdd8d94138"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 27.491333484649658,
|
|
"offered_request_rate": 1.6,
|
|
"request_metrics_sha256": "b7d73a1cbef8ca659f753135a4944cb3023d3a2424a19a741ae2198f39c2d172",
|
|
"request_rate_per_gpu": 0.4,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 17513.570713343794,
|
|
"request_id": 0,
|
|
"tpot_ms": 135.79521453495556,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 17840.282304762335,
|
|
"request_id": 1,
|
|
"tpot_ms": 137.71064841138914,
|
|
"ttft_ms": 351.0299565159147
|
|
},
|
|
{
|
|
"e2e_ms": 17895.63618825912,
|
|
"request_id": 2,
|
|
"tpot_ms": 138.10245652748367,
|
|
"ttft_ms": 356.6242092686922
|
|
},
|
|
{
|
|
"e2e_ms": 17848.77057711112,
|
|
"request_id": 3,
|
|
"tpot_ms": 138.39719968518088,
|
|
"ttft_ms": 272.32621709314975
|
|
},
|
|
{
|
|
"e2e_ms": 17904.1244606079,
|
|
"request_id": 4,
|
|
"tpot_ms": 138.80479077491768,
|
|
"ttft_ms": 275.9160321933569
|
|
},
|
|
{
|
|
"e2e_ms": 17959.478344104686,
|
|
"request_id": 5,
|
|
"tpot_ms": 139.18735062645055,
|
|
"ttft_ms": 282.6848145454632
|
|
},
|
|
{
|
|
"e2e_ms": 18014.921946878916,
|
|
"request_id": 6,
|
|
"tpot_ms": 139.5152186600215,
|
|
"ttft_ms": 296.48917705618504
|
|
},
|
|
{
|
|
"e2e_ms": 18070.36554965315,
|
|
"request_id": 7,
|
|
"tpot_ms": 139.8086106441042,
|
|
"ttft_ms": 314.671997851919
|
|
},
|
|
{
|
|
"e2e_ms": 18125.898871704834,
|
|
"request_id": 8,
|
|
"tpot_ms": 140.0646556798527,
|
|
"ttft_ms": 337.68760036353916
|
|
},
|
|
{
|
|
"e2e_ms": 18079.30241838919,
|
|
"request_id": 9,
|
|
"tpot_ms": 140.21470125635153,
|
|
"ttft_ms": 272.0353588325457
|
|
},
|
|
{
|
|
"e2e_ms": 18134.83860521682,
|
|
"request_id": 10,
|
|
"tpot_ms": 140.33922237580643,
|
|
"ttft_ms": 311.75736348940484
|
|
},
|
|
{
|
|
"e2e_ms": 18190.4645113219,
|
|
"request_id": 11,
|
|
"tpot_ms": 140.37869703173442,
|
|
"ttft_ms": 362.36998829163
|
|
},
|
|
{
|
|
"e2e_ms": 18143.870922782207,
|
|
"request_id": 12,
|
|
"tpot_ms": 140.38698051363622,
|
|
"ttft_ms": 314.7243975504068
|
|
},
|
|
{
|
|
"e2e_ms": 18097.274469466563,
|
|
"request_id": 13,
|
|
"tpot_ms": 140.36074521133403,
|
|
"ttft_ms": 271.4598276271438
|
|
},
|
|
{
|
|
"e2e_ms": 18152.810656294194,
|
|
"request_id": 14,
|
|
"tpot_ms": 140.32455966134754,
|
|
"ttft_ms": 331.5915793030584
|
|
},
|
|
{
|
|
"e2e_ms": 18106.217067754496,
|
|
"request_id": 15,
|
|
"tpot_ms": 140.29231877139563,
|
|
"ttft_ms": 289.0925837872533
|
|
},
|
|
{
|
|
"e2e_ms": 18161.753254582127,
|
|
"request_id": 16,
|
|
"tpot_ms": 140.24009333090902,
|
|
"ttft_ms": 351.2614015566804
|
|
},
|
|
{
|
|
"e2e_ms": 18115.15966604243,
|
|
"request_id": 17,
|
|
"tpot_ms": 140.19912163029534,
|
|
"ttft_ms": 309.87121899492286
|
|
},
|
|
{
|
|
"e2e_ms": 18170.69585287006,
|
|
"request_id": 18,
|
|
"tpot_ms": 140.12856604535176,
|
|
"ttft_ms": 374.3679651103839
|
|
},
|
|
{
|
|
"e2e_ms": 18124.102264330366,
|
|
"request_id": 19,
|
|
"tpot_ms": 140.05569589908265,
|
|
"ttft_ms": 337.0288851468697
|
|
},
|
|
{
|
|
"e2e_ms": 18077.42933293013,
|
|
"request_id": 20,
|
|
"tpot_ms": 139.98073600581233,
|
|
"ttft_ms": 299.87586019196044
|
|
},
|
|
{
|
|
"e2e_ms": 18132.96551975776,
|
|
"request_id": 21,
|
|
"tpot_ms": 139.8495015438229,
|
|
"ttft_ms": 372.0788236922541
|
|
},
|
|
{
|
|
"e2e_ms": 18086.369066442116,
|
|
"request_id": 22,
|
|
"tpot_ms": 139.76199603449132,
|
|
"ttft_ms": 336.5955700617178
|
|
},
|
|
{
|
|
"e2e_ms": 18039.682893849018,
|
|
"request_id": 23,
|
|
"tpot_ms": 139.67091992877667,
|
|
"ttft_ms": 301.4760628943822
|
|
},
|
|
{
|
|
"e2e_ms": 18095.21908067664,
|
|
"request_id": 24,
|
|
"tpot_ms": 139.54200325642663,
|
|
"ttft_ms": 373.38466711045726
|
|
},
|
|
{
|
|
"e2e_ms": 18048.532908083544,
|
|
"request_id": 25,
|
|
"tpot_ms": 139.5517742330953,
|
|
"ttft_ms": 325.4575804804407
|
|
},
|
|
{
|
|
"e2e_ms": 18001.846735490446,
|
|
"request_id": 26,
|
|
"tpot_ms": 139.5610841122856,
|
|
"ttft_ms": 277.589053230173
|
|
},
|
|
{
|
|
"e2e_ms": 18057.290338264673,
|
|
"request_id": 27,
|
|
"tpot_ms": 139.56778701592805,
|
|
"ttft_ms": 332.18138724181046
|
|
},
|
|
{
|
|
"e2e_ms": 18010.60416567157,
|
|
"request_id": 28,
|
|
"tpot_ms": 139.57154350494935,
|
|
"ttft_ms": 285.01814054300564
|
|
},
|
|
{
|
|
"e2e_ms": 18066.06100963866,
|
|
"request_id": 29,
|
|
"tpot_ms": 139.5730606683175,
|
|
"ttft_ms": 340.2823047623365
|
|
},
|
|
{
|
|
"e2e_ms": 18019.37483704556,
|
|
"request_id": 30,
|
|
"tpot_ms": 139.57447357032456,
|
|
"ttft_ms": 293.41669361433986
|
|
},
|
|
{
|
|
"e2e_ms": 18074.831681012645,
|
|
"request_id": 31,
|
|
"tpot_ms": 139.57528428268915,
|
|
"ttft_ms": 348.7705771111216
|
|
},
|
|
{
|
|
"e2e_ms": 18028.145508419548,
|
|
"request_id": 32,
|
|
"tpot_ms": 139.57669718469626,
|
|
"ttft_ms": 301.904965963125
|
|
},
|
|
{
|
|
"e2e_ms": 18083.678830471228,
|
|
"request_id": 33,
|
|
"tpot_ms": 139.5781100867033,
|
|
"ttft_ms": 357.25884945990674
|
|
},
|
|
{
|
|
"e2e_ms": 18036.99265787813,
|
|
"request_id": 34,
|
|
"tpot_ms": 139.57881653770684,
|
|
"ttft_ms": 310.48295758936104
|
|
},
|
|
{
|
|
"e2e_ms": 18092.528844705754,
|
|
"request_id": 35,
|
|
"tpot_ms": 139.57954554600127,
|
|
"ttft_ms": 365.9265603635937
|
|
},
|
|
{
|
|
"e2e_ms": 17876.05367572695,
|
|
"request_id": 36,
|
|
"tpot_ms": 138.24262431461773,
|
|
"ttft_ms": 319.240387770499
|
|
},
|
|
{
|
|
"e2e_ms": 17659.21004164318,
|
|
"request_id": 37,
|
|
"tpot_ms": 136.90209533219152,
|
|
"ttft_ms": 272.64393445485524
|
|
},
|
|
{
|
|
"e2e_ms": 17543.75228976072,
|
|
"request_id": 38,
|
|
"tpot_ms": 135.55570892326125,
|
|
"ttft_ms": 328.1772565065388
|
|
},
|
|
{
|
|
"e2e_ms": 17326.367174433202,
|
|
"request_id": 39,
|
|
"tpot_ms": 134.21089375170362,
|
|
"ttft_ms": 281.5836679668422
|
|
},
|
|
{
|
|
"e2e_ms": 17233.832476815856,
|
|
"request_id": 40,
|
|
"tpot_ms": 133.04427482475538,
|
|
"ttft_ms": 337.20957407192384
|
|
},
|
|
{
|
|
"e2e_ms": 17034.265611203216,
|
|
"request_id": 41,
|
|
"tpot_ms": 131.83976083205505,
|
|
"ttft_ms": 290.6159855322272
|
|
},
|
|
{
|
|
"e2e_ms": 16940.502121615402,
|
|
"request_id": 42,
|
|
"tpot_ms": 130.66419538607474,
|
|
"ttft_ms": 346.14930758391085
|
|
},
|
|
{
|
|
"e2e_ms": 16741.97064984836,
|
|
"request_id": 43,
|
|
"tpot_ms": 129.46783410082003,
|
|
"ttft_ms": 299.5557190442142
|
|
},
|
|
{
|
|
"e2e_ms": 16645.559306371404,
|
|
"request_id": 44,
|
|
"tpot_ms": 128.27139685432724,
|
|
"ttft_ms": 355.0919058718449
|
|
},
|
|
{
|
|
"e2e_ms": 16443.232533552808,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.04515130882409,
|
|
"ttft_ms": 308.4983173321483
|
|
},
|
|
{
|
|
"e2e_ms": 16342.299705615802,
|
|
"request_id": 46,
|
|
"tpot_ms": 125.81311182248837,
|
|
"ttft_ms": 364.034504159779
|
|
},
|
|
{
|
|
"e2e_ms": 16135.159559133626,
|
|
"request_id": 47,
|
|
"tpot_ms": 124.54896569695704,
|
|
"ttft_ms": 317.44091562008236
|
|
},
|
|
{
|
|
"e2e_ms": 16031.7032146663,
|
|
"request_id": 48,
|
|
"tpot_ms": 123.29695174036006,
|
|
"ttft_ms": 372.9903436405735
|
|
},
|
|
{
|
|
"e2e_ms": 15823.308920667878,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.0236594458299,
|
|
"ttft_ms": 326.3041710474788
|
|
},
|
|
{
|
|
"e2e_ms": 15613.44809689394,
|
|
"request_id": 50,
|
|
"tpot_ms": 120.73809066445794,
|
|
"ttft_ms": 279.71058250778214
|
|
},
|
|
{
|
|
"e2e_ms": 15508.031714506984,
|
|
"request_id": 51,
|
|
"tpot_ms": 119.47077015706711,
|
|
"ttft_ms": 335.2439045594622
|
|
},
|
|
{
|
|
"e2e_ms": 15292.981047367362,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.14506547559841,
|
|
"ttft_ms": 288.5577319663639
|
|
},
|
|
{
|
|
"e2e_ms": 15175.360910096466,
|
|
"request_id": 53,
|
|
"tpot_ms": 116.78162985277542,
|
|
"ttft_ms": 344.09391879398754
|
|
},
|
|
{
|
|
"e2e_ms": 14945.589828664517,
|
|
"request_id": 54,
|
|
"tpot_ms": 115.34001639735139,
|
|
"ttft_ms": 297.4077462008893
|
|
},
|
|
{
|
|
"e2e_ms": 14803.64975337431,
|
|
"request_id": 55,
|
|
"tpot_ms": 113.78581420786767,
|
|
"ttft_ms": 352.85134897511483
|
|
},
|
|
{
|
|
"e2e_ms": 14557.342221274233,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.21399247946627,
|
|
"ttft_ms": 306.16517638201657
|
|
},
|
|
{
|
|
"e2e_ms": 14399.594833150197,
|
|
"request_id": 57,
|
|
"tpot_ms": 110.5352189984338,
|
|
"ttft_ms": 361.6220203491025
|
|
},
|
|
{
|
|
"e2e_ms": 14144.625862747695,
|
|
"request_id": 58,
|
|
"tpot_ms": 108.89519696843851,
|
|
"ttft_ms": 314.93584775600425
|
|
},
|
|
{
|
|
"e2e_ms": 13973.496543699825,
|
|
"request_id": 59,
|
|
"tpot_ms": 107.1110539525727,
|
|
"ttft_ms": 370.3926917230902
|
|
},
|
|
{
|
|
"e2e_ms": 13708.4999588278,
|
|
"request_id": 60,
|
|
"tpot_ms": 105.3920743283292,
|
|
"ttft_ms": 323.70651912999193
|
|
},
|
|
{
|
|
"e2e_ms": 13445.970803422128,
|
|
"request_id": 61,
|
|
"tpot_ms": 103.6925232825609,
|
|
"ttft_ms": 277.02034653689367
|
|
},
|
|
{
|
|
"e2e_ms": 13273.580023880399,
|
|
"request_id": 62,
|
|
"tpot_ms": 101.89784531725846,
|
|
"ttft_ms": 332.5536685885737
|
|
},
|
|
{
|
|
"e2e_ms": 13000.702540117643,
|
|
"request_id": 63,
|
|
"tpot_ms": 100.1168113710407,
|
|
"ttft_ms": 285.86749599547545
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.203125,
|
|
"passed": 13
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "850087527f8ec5b333ae4bb00c5837c85f7dafca77de3d8c80820e071da8dfbb"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 35.55747365951538,
|
|
"offered_request_rate": 2.4,
|
|
"request_metrics_sha256": "3b42135fbabbd54647284962ccc9fdf0b900fbc0b6ed0bdecf03477d5c409d18",
|
|
"request_rate_per_gpu": 0.6,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 21942.173036302996,
|
|
"request_id": 0,
|
|
"tpot_ms": 170.66609896770518,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 22008.341136862466,
|
|
"request_id": 1,
|
|
"tpot_ms": 170.96613901739173,
|
|
"ttft_ms": 295.64148165371654
|
|
},
|
|
{
|
|
"e2e_ms": 22074.54102422534,
|
|
"request_id": 2,
|
|
"tpot_ms": 171.22660942224667,
|
|
"ttft_ms": 328.7616276000109
|
|
},
|
|
{
|
|
"e2e_ms": 22036.227907899276,
|
|
"request_id": 3,
|
|
"tpot_ms": 171.37344968894675,
|
|
"ttft_ms": 271.7997974030406
|
|
},
|
|
{
|
|
"e2e_ms": 22102.42779526115,
|
|
"request_id": 4,
|
|
"tpot_ms": 171.64131409153669,
|
|
"ttft_ms": 303.98090563598765
|
|
},
|
|
{
|
|
"e2e_ms": 22168.65946942643,
|
|
"request_id": 5,
|
|
"tpot_ms": 171.89659303259273,
|
|
"ttft_ms": 337.79215428715537
|
|
},
|
|
{
|
|
"e2e_ms": 22130.37813990278,
|
|
"request_id": 6,
|
|
"tpot_ms": 172.0281077294899,
|
|
"ttft_ms": 282.80845825756097
|
|
},
|
|
{
|
|
"e2e_ms": 22365.99994575454,
|
|
"request_id": 7,
|
|
"tpot_ms": 173.57085404199856,
|
|
"ttft_ms": 322.5014824207202
|
|
},
|
|
{
|
|
"e2e_ms": 22158.3592864644,
|
|
"request_id": 8,
|
|
"tpot_ms": 172.3496956452424,
|
|
"ttft_ms": 269.9479395186155
|
|
},
|
|
{
|
|
"e2e_ms": 22393.981092316164,
|
|
"request_id": 9,
|
|
"tpot_ms": 173.8419366523574,
|
|
"ttft_ms": 316.0551374667744
|
|
},
|
|
{
|
|
"e2e_ms": 22460.243568400445,
|
|
"request_id": 10,
|
|
"tpot_ms": 173.96590922572528,
|
|
"ttft_ms": 366.57309673333316
|
|
},
|
|
{
|
|
"e2e_ms": 22252.602909110316,
|
|
"request_id": 11,
|
|
"tpot_ms": 172.6883751510306,
|
|
"ttft_ms": 321.1792649294285
|
|
},
|
|
{
|
|
"e2e_ms": 22044.894176229613,
|
|
"request_id": 12,
|
|
"tpot_ms": 171.40112664047544,
|
|
"ttft_ms": 276.9510928892318
|
|
},
|
|
{
|
|
"e2e_ms": 21941.653543442477,
|
|
"request_id": 13,
|
|
"tpot_ms": 170.1065182287643,
|
|
"ttft_ms": 338.1257283894126
|
|
},
|
|
{
|
|
"e2e_ms": 21734.138079618475,
|
|
"request_id": 14,
|
|
"tpot_ms": 168.80971544527205,
|
|
"ttft_ms": 295.30421806892537
|
|
},
|
|
{
|
|
"e2e_ms": 21631.119179231846,
|
|
"request_id": 15,
|
|
"tpot_ms": 167.51267286797687,
|
|
"ttft_ms": 357.0097249987798
|
|
},
|
|
{
|
|
"e2e_ms": 21420.354658102242,
|
|
"request_id": 16,
|
|
"tpot_ms": 166.18662353135022,
|
|
"ttft_ms": 314.65346962076256
|
|
},
|
|
{
|
|
"e2e_ms": 21209.648295563282,
|
|
"request_id": 17,
|
|
"tpot_ms": 164.85776701341018,
|
|
"ttft_ms": 272.7118848601879
|
|
},
|
|
{
|
|
"e2e_ms": 21101.69867612793,
|
|
"request_id": 18,
|
|
"tpot_ms": 163.50974377121332,
|
|
"ttft_ms": 335.9612171838391
|
|
},
|
|
{
|
|
"e2e_ms": 20890.778369055995,
|
|
"request_id": 19,
|
|
"tpot_ms": 162.16628083276893,
|
|
"ttft_ms": 295.66070329433813
|
|
},
|
|
{
|
|
"e2e_ms": 20782.558056179907,
|
|
"request_id": 20,
|
|
"tpot_ms": 160.7985342727971,
|
|
"ttft_ms": 361.14420353467835
|
|
},
|
|
{
|
|
"e2e_ms": 20571.512469390585,
|
|
"request_id": 21,
|
|
"tpot_ms": 159.44598375979533,
|
|
"ttft_ms": 321.8725318965792
|
|
},
|
|
{
|
|
"e2e_ms": 20360.405278761264,
|
|
"request_id": 22,
|
|
"tpot_ms": 158.09565605925098,
|
|
"ttft_ms": 282.2569592363866
|
|
},
|
|
{
|
|
"e2e_ms": 20251.586130359967,
|
|
"request_id": 23,
|
|
"tpot_ms": 156.71227679254048,
|
|
"ttft_ms": 349.12697770732314
|
|
},
|
|
{
|
|
"e2e_ms": 20042.10448060375,
|
|
"request_id": 24,
|
|
"tpot_ms": 155.37142770924103,
|
|
"ttft_ms": 309.93316153013774
|
|
},
|
|
{
|
|
"e2e_ms": 19934.008109113514,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.05960281316982,
|
|
"ttft_ms": 368.43855184094565
|
|
},
|
|
{
|
|
"e2e_ms": 19722.984661067087,
|
|
"request_id": 26,
|
|
"tpot_ms": 152.73931080974432,
|
|
"ttft_ms": 325.0921882295579
|
|
},
|
|
{
|
|
"e2e_ms": 19511.768043076878,
|
|
"request_id": 27,
|
|
"tpot_ms": 151.41621155599998,
|
|
"ttft_ms": 281.9091754648788
|
|
},
|
|
{
|
|
"e2e_ms": 19403.203575383635,
|
|
"request_id": 28,
|
|
"tpot_ms": 150.09414530000853,
|
|
"ttft_ms": 341.2471222825513
|
|
},
|
|
{
|
|
"e2e_ms": 19191.714653347688,
|
|
"request_id": 29,
|
|
"tpot_ms": 148.76487274343572,
|
|
"ttft_ms": 298.57581493135046
|
|
},
|
|
{
|
|
"e2e_ms": 19082.708845359608,
|
|
"request_id": 30,
|
|
"tpot_ms": 147.43434256455518,
|
|
"ttft_ms": 358.5473396611025
|
|
},
|
|
{
|
|
"e2e_ms": 18871.078709760895,
|
|
"request_id": 31,
|
|
"tpot_ms": 146.10277278695904,
|
|
"ttft_ms": 316.0265658170971
|
|
},
|
|
{
|
|
"e2e_ms": 18659.370447496967,
|
|
"request_id": 32,
|
|
"tpot_ms": 144.7695321688903,
|
|
"ttft_ms": 273.6398620479008
|
|
},
|
|
{
|
|
"e2e_ms": 18549.637648096734,
|
|
"request_id": 33,
|
|
"tpot_ms": 143.43132538741386,
|
|
"ttft_ms": 333.85932389517234
|
|
},
|
|
{
|
|
"e2e_ms": 18337.808235788358,
|
|
"request_id": 34,
|
|
"tpot_ms": 142.0958670025312,
|
|
"ttft_ms": 291.63312646689656
|
|
},
|
|
{
|
|
"e2e_ms": 18227.80864738452,
|
|
"request_id": 35,
|
|
"tpot_ms": 140.75352098043697,
|
|
"ttft_ms": 352.1114828690255
|
|
},
|
|
{
|
|
"e2e_ms": 18015.40726100407,
|
|
"request_id": 36,
|
|
"tpot_ms": 139.41149279948888,
|
|
"ttft_ms": 310.14767546898446
|
|
},
|
|
{
|
|
"e2e_ms": 17904.857868774245,
|
|
"request_id": 37,
|
|
"tpot_ms": 138.06211837292886,
|
|
"ttft_ms": 370.9688354122758
|
|
},
|
|
{
|
|
"e2e_ms": 17692.008101355255,
|
|
"request_id": 38,
|
|
"tpot_ms": 136.71510258553306,
|
|
"ttft_ms": 329.1900729925583
|
|
},
|
|
{
|
|
"e2e_ms": 17479.148877024498,
|
|
"request_id": 39,
|
|
"tpot_ms": 135.36749298254742,
|
|
"ttft_ms": 287.47726824097344
|
|
},
|
|
{
|
|
"e2e_ms": 17381.961391787096,
|
|
"request_id": 40,
|
|
"tpot_ms": 134.1089349239123,
|
|
"ttft_ms": 350.1266564502323
|
|
},
|
|
{
|
|
"e2e_ms": 17178.01129231477,
|
|
"request_id": 41,
|
|
"tpot_ms": 132.8327622110564,
|
|
"ttft_ms": 308.2504915106092
|
|
},
|
|
{
|
|
"e2e_ms": 17080.086531895085,
|
|
"request_id": 42,
|
|
"tpot_ms": 131.5805170059729,
|
|
"ttft_ms": 369.36087213652866
|
|
},
|
|
{
|
|
"e2e_ms": 16876.654129344566,
|
|
"request_id": 43,
|
|
"tpot_ms": 130.30640250229436,
|
|
"ttft_ms": 327.74101155318294
|
|
},
|
|
{
|
|
"e2e_ms": 16671.422925287778,
|
|
"request_id": 44,
|
|
"tpot_ms": 129.0178430005707,
|
|
"ttft_ms": 286.15686421529676
|
|
},
|
|
{
|
|
"e2e_ms": 16571.761179006833,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.74972161297816,
|
|
"ttft_ms": 347.5465341586066
|
|
},
|
|
{
|
|
"e2e_ms": 16364.721381165033,
|
|
"request_id": 46,
|
|
"tpot_ms": 126.44572566423672,
|
|
"ttft_ms": 306.1142218069683
|
|
},
|
|
{
|
|
"e2e_ms": 16261.449604637399,
|
|
"request_id": 47,
|
|
"tpot_ms": 125.14700916940275,
|
|
"ttft_ms": 367.77944012325037
|
|
},
|
|
{
|
|
"e2e_ms": 16053.400400183471,
|
|
"request_id": 48,
|
|
"tpot_ms": 123.8340136000885,
|
|
"ttft_ms": 326.48067297223093
|
|
},
|
|
{
|
|
"e2e_ms": 15845.03658651726,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.50141376236263,
|
|
"ttft_ms": 287.35703869720555
|
|
},
|
|
{
|
|
"e2e_ms": 15739.724302020804,
|
|
"request_id": 50,
|
|
"tpot_ms": 121.15600683324593,
|
|
"ttft_ms": 352.9114341985711
|
|
},
|
|
{
|
|
"e2e_ms": 15530.891082399023,
|
|
"request_id": 51,
|
|
"tpot_ms": 119.81946138926746,
|
|
"ttft_ms": 313.8194859620569
|
|
},
|
|
{
|
|
"e2e_ms": 15319.199082162215,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.45427332697808,
|
|
"ttft_ms": 275.5063696359983
|
|
},
|
|
{
|
|
"e2e_ms": 15206.960333133673,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.04949498375757,
|
|
"ttft_ms": 341.6744701964625
|
|
},
|
|
{
|
|
"e2e_ms": 14987.9081257507,
|
|
"request_id": 54,
|
|
"tpot_ms": 115.62635253449054,
|
|
"ttft_ms": 303.36135387040383
|
|
},
|
|
{
|
|
"e2e_ms": 14861.077413909577,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.10642655651417,
|
|
"ttft_ms": 369.5612412322795
|
|
},
|
|
{
|
|
"e2e_ms": 14633.756981193535,
|
|
"request_id": 56,
|
|
"tpot_ms": 112.61817997075843,
|
|
"ttft_ms": 331.2481249072157
|
|
},
|
|
{
|
|
"e2e_ms": 14403.991359276923,
|
|
"request_id": 57,
|
|
"tpot_ms": 111.11042963695559,
|
|
"ttft_ms": 292.9667953835633
|
|
},
|
|
{
|
|
"e2e_ms": 14264.847964808048,
|
|
"request_id": 58,
|
|
"tpot_ms": 109.49330311228506,
|
|
"ttft_ms": 359.1984695478452
|
|
},
|
|
{
|
|
"e2e_ms": 14029.729570522895,
|
|
"request_id": 59,
|
|
"tpot_ms": 107.94340496454889,
|
|
"ttft_ms": 320.91714002518756
|
|
},
|
|
{
|
|
"e2e_ms": 13793.064611419886,
|
|
"request_id": 60,
|
|
"tpot_ms": 106.38108660628622,
|
|
"ttft_ms": 282.66661242153646
|
|
},
|
|
{
|
|
"e2e_ms": 13648.251078198633,
|
|
"request_id": 61,
|
|
"tpot_ms": 104.71931331978594,
|
|
"ttft_ms": 348.89828658581837
|
|
},
|
|
{
|
|
"e2e_ms": 13412.62809971594,
|
|
"request_id": 62,
|
|
"tpot_ms": 103.16519953332896,
|
|
"ttft_ms": 310.647758983162
|
|
},
|
|
{
|
|
"e2e_ms": 13260.053320226874,
|
|
"request_id": 63,
|
|
"tpot_ms": 101.44207153668842,
|
|
"ttft_ms": 376.9102350674451
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.203125,
|
|
"passed": 13
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.546875,
|
|
"passed": 35
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "0b9f30d883de83b4af69dd114bd5d2bb974b7fd1f01d45f913bf5049e4b72a86"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"elapsed_seconds": 33.80778503417969,
|
|
"offered_request_rate": 3.2,
|
|
"request_metrics_sha256": "bfba1ae71cfb6d0595614b14bc49ac47591bfad6792be0a3d98d2c98b16012e8",
|
|
"request_rate_per_gpu": 0.8,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 24003.81066522886,
|
|
"request_id": 0,
|
|
"tpot_ms": 186.89946612460176,
|
|
"ttft_ms": 267.578467404437
|
|
},
|
|
{
|
|
"e2e_ms": 23901.058156177503,
|
|
"request_id": 1,
|
|
"tpot_ms": 185.74139588389912,
|
|
"ttft_ms": 311.9008789223171
|
|
},
|
|
{
|
|
"e2e_ms": 23798.190109844723,
|
|
"request_id": 2,
|
|
"tpot_ms": 184.56225305811301,
|
|
"ttft_ms": 358.7839714643697
|
|
},
|
|
{
|
|
"e2e_ms": 23590.44639829378,
|
|
"request_id": 3,
|
|
"tpot_ms": 183.2673714648551,
|
|
"ttft_ms": 315.49022225718517
|
|
},
|
|
{
|
|
"e2e_ms": 23382.70131446869,
|
|
"request_id": 4,
|
|
"tpot_ms": 181.9732705315891,
|
|
"ttft_ms": 272.09595695687216
|
|
},
|
|
{
|
|
"e2e_ms": 23279.64954297851,
|
|
"request_id": 5,
|
|
"tpot_ms": 180.78914255292864,
|
|
"ttft_ms": 319.4284387565705
|
|
},
|
|
{
|
|
"e2e_ms": 23071.973865900134,
|
|
"request_id": 6,
|
|
"tpot_ms": 179.49398625037213,
|
|
"ttft_ms": 276.2376121028707
|
|
},
|
|
{
|
|
"e2e_ms": 22969.022726489202,
|
|
"request_id": 7,
|
|
"tpot_ms": 178.2861180199918,
|
|
"ttft_ms": 326.6857379502426
|
|
},
|
|
{
|
|
"e2e_ms": 22761.29942646063,
|
|
"request_id": 8,
|
|
"tpot_ms": 176.98924979786668,
|
|
"ttft_ms": 283.66470213156344
|
|
},
|
|
{
|
|
"e2e_ms": 22658.264587165988,
|
|
"request_id": 9,
|
|
"tpot_ms": 175.7551174352214,
|
|
"ttft_ms": 337.36467289287384
|
|
},
|
|
{
|
|
"e2e_ms": 22450.304224673095,
|
|
"request_id": 10,
|
|
"tpot_ms": 174.4550575162989,
|
|
"ttft_ms": 294.51192010313275
|
|
},
|
|
{
|
|
"e2e_ms": 22346.821408551063,
|
|
"request_id": 11,
|
|
"tpot_ms": 173.1774539652428,
|
|
"ttft_ms": 353.28475496522805
|
|
},
|
|
{
|
|
"e2e_ms": 22138.79812545183,
|
|
"request_id": 12,
|
|
"tpot_ms": 171.87427427697472,
|
|
"ttft_ms": 310.7652922760442
|
|
},
|
|
{
|
|
"e2e_ms": 22035.248814705075,
|
|
"request_id": 13,
|
|
"tpot_ms": 170.57508935568035,
|
|
"ttft_ms": 372.21246653366967
|
|
},
|
|
{
|
|
"e2e_ms": 21827.324416126074,
|
|
"request_id": 14,
|
|
"tpot_ms": 169.27188182433355,
|
|
"ttft_ms": 329.79542443571484
|
|
},
|
|
{
|
|
"e2e_ms": 21619.373671552865,
|
|
"request_id": 15,
|
|
"tpot_ms": 167.96821838814947,
|
|
"ttft_ms": 287.40993625788303
|
|
},
|
|
{
|
|
"e2e_ms": 21512.77581709026,
|
|
"request_id": 16,
|
|
"tpot_ms": 166.64216905152279,
|
|
"ttft_ms": 349.2203475468658
|
|
},
|
|
{
|
|
"e2e_ms": 21303.255969153783,
|
|
"request_id": 17,
|
|
"tpot_ms": 165.32482604305434,
|
|
"ttft_ms": 307.003061685883
|
|
},
|
|
{
|
|
"e2e_ms": 21196.567333974876,
|
|
"request_id": 18,
|
|
"tpot_ms": 163.99164288866731,
|
|
"ttft_ms": 369.62868711412875
|
|
},
|
|
{
|
|
"e2e_ms": 20986.94051377241,
|
|
"request_id": 19,
|
|
"tpot_ms": 162.67002407488405,
|
|
"ttft_ms": 327.84745626213316
|
|
},
|
|
{
|
|
"e2e_ms": 20777.255965035714,
|
|
"request_id": 20,
|
|
"tpot_ms": 161.34765321373814,
|
|
"ttft_ms": 286.10400689097213
|
|
},
|
|
{
|
|
"e2e_ms": 20670.377044913395,
|
|
"request_id": 21,
|
|
"tpot_ms": 159.9951027007364,
|
|
"ttft_ms": 350.99900191987297
|
|
},
|
|
{
|
|
"e2e_ms": 20460.656782932234,
|
|
"request_id": 22,
|
|
"tpot_ms": 158.67196467076286,
|
|
"ttft_ms": 309.3172697453532
|
|
},
|
|
{
|
|
"e2e_ms": 20353.38846177537,
|
|
"request_id": 23,
|
|
"tpot_ms": 157.31793948495124,
|
|
"ttft_ms": 374.01014718656
|
|
},
|
|
{
|
|
"e2e_ms": 20144.48097023076,
|
|
"request_id": 24,
|
|
"tpot_ms": 155.99981541047092,
|
|
"ttft_ms": 332.5044131009545
|
|
},
|
|
{
|
|
"e2e_ms": 19934.83773528968,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.6729368346084,
|
|
"ttft_ms": 291.37475729441144
|
|
},
|
|
{
|
|
"e2e_ms": 19827.980953909257,
|
|
"request_id": 26,
|
|
"tpot_ms": 153.35264483118294,
|
|
"ttft_ms": 352.1950603490236
|
|
},
|
|
{
|
|
"e2e_ms": 19618.20597824765,
|
|
"request_id": 27,
|
|
"tpot_ms": 152.02412197401114,
|
|
"ttft_ms": 311.1424875482367
|
|
},
|
|
{
|
|
"e2e_ms": 19511.107444230154,
|
|
"request_id": 28,
|
|
"tpot_ms": 150.6975599886974,
|
|
"ttft_ms": 372.5173256655836
|
|
},
|
|
{
|
|
"e2e_ms": 19301.196316545185,
|
|
"request_id": 29,
|
|
"tpot_ms": 149.36537894653213,
|
|
"ttft_ms": 331.7931903356044
|
|
},
|
|
{
|
|
"e2e_ms": 19091.249936104825,
|
|
"request_id": 30,
|
|
"tpot_ms": 148.03267778881258,
|
|
"ttft_ms": 291.0998569256247
|
|
},
|
|
{
|
|
"e2e_ms": 18983.78646717311,
|
|
"request_id": 31,
|
|
"tpot_ms": 146.7011080112165,
|
|
"ttft_ms": 352.7457497486193
|
|
},
|
|
{
|
|
"e2e_ms": 18773.765669374145,
|
|
"request_id": 32,
|
|
"tpot_ms": 145.36784483585674,
|
|
"ttft_ms": 312.0493752203348
|
|
},
|
|
{
|
|
"e2e_ms": 18665.888247551986,
|
|
"request_id": 33,
|
|
"tpot_ms": 144.03124523287858,
|
|
"ttft_ms": 373.9201029764097
|
|
},
|
|
{
|
|
"e2e_ms": 18455.8068747313,
|
|
"request_id": 34,
|
|
"tpot_ms": 142.6968275070543,
|
|
"ttft_ms": 333.30978133540157
|
|
},
|
|
{
|
|
"e2e_ms": 18245.529234152025,
|
|
"request_id": 35,
|
|
"tpot_ms": 141.3602386069844,
|
|
"ttft_ms": 292.7789310650048
|
|
},
|
|
{
|
|
"e2e_ms": 18137.294514438578,
|
|
"request_id": 36,
|
|
"tpot_ms": 140.01821042603632,
|
|
"ttft_ms": 354.98179033196385
|
|
},
|
|
{
|
|
"e2e_ms": 17926.83360591763,
|
|
"request_id": 37,
|
|
"tpot_ms": 138.67967407469433,
|
|
"ttft_ms": 314.5149984314486
|
|
},
|
|
{
|
|
"e2e_ms": 17716.242055541137,
|
|
"request_id": 38,
|
|
"tpot_ms": 137.3398665125213,
|
|
"ttft_ms": 274.0790084509328
|
|
},
|
|
{
|
|
"e2e_ms": 17607.549497877386,
|
|
"request_id": 39,
|
|
"tpot_ms": 135.99225690953574,
|
|
"ttft_ms": 336.53287036634794
|
|
},
|
|
{
|
|
"e2e_ms": 17401.54255835392,
|
|
"request_id": 40,
|
|
"tpot_ms": 134.6880568492608,
|
|
"ttft_ms": 296.1593384978016
|
|
},
|
|
{
|
|
"e2e_ms": 17301.7591255476,
|
|
"request_id": 41,
|
|
"tpot_ms": 133.41188413640484,
|
|
"ttft_ms": 358.4498402241838
|
|
},
|
|
{
|
|
"e2e_ms": 17095.506427630044,
|
|
"request_id": 42,
|
|
"tpot_ms": 132.10522228512968,
|
|
"ttft_ms": 318.1431974185731
|
|
},
|
|
{
|
|
"e2e_ms": 16889.623559688287,
|
|
"request_id": 43,
|
|
"tpot_ms": 130.8012299461049,
|
|
"ttft_ms": 277.86735653296194
|
|
},
|
|
{
|
|
"e2e_ms": 16788.559022297493,
|
|
"request_id": 44,
|
|
"tpot_ms": 129.5126704443812,
|
|
"ttft_ms": 340.449875861081
|
|
},
|
|
{
|
|
"e2e_ms": 16581.727329092846,
|
|
"request_id": 45,
|
|
"tpot_ms": 128.200721970688,
|
|
"ttft_ms": 300.23563881547057
|
|
},
|
|
{
|
|
"e2e_ms": 16478.854197918037,
|
|
"request_id": 46,
|
|
"tpot_ms": 126.8967260219465,
|
|
"ttft_ms": 362.9699931308306
|
|
},
|
|
{
|
|
"e2e_ms": 16270.819161297495,
|
|
"request_id": 47,
|
|
"tpot_ms": 125.57481733363996,
|
|
"ttft_ms": 322.8173599252209
|
|
},
|
|
{
|
|
"e2e_ms": 16062.627892404027,
|
|
"request_id": 48,
|
|
"tpot_ms": 124.25122687214136,
|
|
"ttft_ms": 282.72207964207576
|
|
},
|
|
{
|
|
"e2e_ms": 15958.430745404816,
|
|
"request_id": 49,
|
|
"tpot_ms": 122.91862703441548,
|
|
"ttft_ms": 347.7651120340504
|
|
},
|
|
{
|
|
"e2e_ms": 15749.715539461335,
|
|
"request_id": 50,
|
|
"tpot_ms": 121.58574387256766,
|
|
"ttft_ms": 308.3260676452415
|
|
},
|
|
{
|
|
"e2e_ms": 15645.048986506545,
|
|
"request_id": 51,
|
|
"tpot_ms": 120.24919842858912,
|
|
"ttft_ms": 373.40078607572735
|
|
},
|
|
{
|
|
"e2e_ms": 15435.036319721643,
|
|
"request_id": 52,
|
|
"tpot_ms": 118.8997102377205,
|
|
"ttft_ms": 334.77311953113895
|
|
},
|
|
{
|
|
"e2e_ms": 15224.012292267464,
|
|
"request_id": 53,
|
|
"tpot_ms": 117.54201604221191,
|
|
"ttft_ms": 296.1762549065519
|
|
},
|
|
{
|
|
"e2e_ms": 15109.12675155149,
|
|
"request_id": 54,
|
|
"tpot_ms": 116.11789378026353,
|
|
"ttft_ms": 362.1542414580219
|
|
},
|
|
{
|
|
"e2e_ms": 14893.238736493451,
|
|
"request_id": 55,
|
|
"tpot_ms": 114.72141540015762,
|
|
"ttft_ms": 323.6189806734337
|
|
},
|
|
{
|
|
"e2e_ms": 14675.411853468433,
|
|
"request_id": 56,
|
|
"tpot_ms": 113.30942780834319,
|
|
"ttft_ms": 285.1145218088469
|
|
},
|
|
{
|
|
"e2e_ms": 14549.812898218817,
|
|
"request_id": 57,
|
|
"tpot_ms": 111.79839094375423,
|
|
"ttft_ms": 351.4172483620292
|
|
},
|
|
{
|
|
"e2e_ms": 14329.820655618192,
|
|
"request_id": 58,
|
|
"tpot_ms": 110.36886820693505,
|
|
"ttft_ms": 312.97439333744137
|
|
},
|
|
{
|
|
"e2e_ms": 14108.094791808619,
|
|
"request_id": 59,
|
|
"tpot_ms": 108.92545237461232,
|
|
"ttft_ms": 274.56234023285475
|
|
},
|
|
{
|
|
"e2e_ms": 13975.596499372607,
|
|
"request_id": 60,
|
|
"tpot_ms": 107.35951607044248,
|
|
"ttft_ms": 340.93795842641225
|
|
},
|
|
{
|
|
"e2e_ms": 13753.714210521188,
|
|
"request_id": 61,
|
|
"tpot_ms": 105.91438347527057,
|
|
"ttft_ms": 302.58750916182464
|
|
},
|
|
{
|
|
"e2e_ms": 13622.257898704496,
|
|
"request_id": 62,
|
|
"tpot_ms": 104.35568588071192,
|
|
"ttft_ms": 369.08579185408286
|
|
},
|
|
{
|
|
"e2e_ms": 13397.788527763809,
|
|
"request_id": 63,
|
|
"tpot_ms": 102.88969749082138,
|
|
"ttft_ms": 330.7969464294942
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.1875,
|
|
"passed": 12
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.546875,
|
|
"passed": 35
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.90625,
|
|
"passed": 58
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "96c4e04e8f133e66449a5567fcde568f27af44bd9f5d1cadeade66b771a8f0f3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"loads": [
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 778.7386267185211,
|
|
"offered_request_rate": 0.1,
|
|
"request_metrics_sha256": "1146954f1d73891536473385a3ecb02b14b7eaeee231291f2cbe6f9d4d222414",
|
|
"request_rate_per_gpu": 0.0125,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12579.084570749343,
|
|
"request_id": 0,
|
|
"tpot_ms": 96.75132177962064,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 12682.66705927438,
|
|
"request_id": 1,
|
|
"tpot_ms": 96.90664947750408,
|
|
"ttft_ms": 375.5225756313614
|
|
},
|
|
{
|
|
"e2e_ms": 12692.014437909165,
|
|
"request_id": 2,
|
|
"tpot_ms": 96.91832475611541,
|
|
"ttft_ms": 383.3871938825091
|
|
},
|
|
{
|
|
"e2e_ms": 12605.650216029226,
|
|
"request_id": 3,
|
|
"tpot_ms": 96.9198422551915,
|
|
"ttft_ms": 296.8302496199051
|
|
},
|
|
{
|
|
"e2e_ms": 12614.259349375281,
|
|
"request_id": 4,
|
|
"tpot_ms": 96.9125118011697,
|
|
"ttft_ms": 306.3703506267288
|
|
},
|
|
{
|
|
"e2e_ms": 12622.868482721336,
|
|
"request_id": 5,
|
|
"tpot_ms": 96.9125118011697,
|
|
"ttft_ms": 314.97948397278464
|
|
},
|
|
{
|
|
"e2e_ms": 12631.47761606615,
|
|
"request_id": 6,
|
|
"tpot_ms": 96.91251180115991,
|
|
"ttft_ms": 323.5886173188405
|
|
},
|
|
{
|
|
"e2e_ms": 12640.086749411268,
|
|
"request_id": 7,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 332.19775066687873
|
|
},
|
|
{
|
|
"e2e_ms": 12648.695882756385,
|
|
"request_id": 8,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 340.80688401199666
|
|
},
|
|
{
|
|
"e2e_ms": 12657.305016101503,
|
|
"request_id": 9,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 349.4160173571146
|
|
},
|
|
{
|
|
"e2e_ms": 12665.91414944662,
|
|
"request_id": 10,
|
|
"tpot_ms": 96.9125118011369,
|
|
"ttft_ms": 358.0251507022325
|
|
},
|
|
{
|
|
"e2e_ms": 12674.523282791739,
|
|
"request_id": 11,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 366.63428404735043
|
|
},
|
|
{
|
|
"e2e_ms": 12683.132416062932,
|
|
"request_id": 12,
|
|
"tpot_ms": 96.91251180055482,
|
|
"ttft_ms": 375.24341739246836
|
|
},
|
|
{
|
|
"e2e_ms": 12692.479794575775,
|
|
"request_id": 13,
|
|
"tpot_ms": 96.91832475507798,
|
|
"ttft_ms": 383.85255068087076
|
|
},
|
|
{
|
|
"e2e_ms": 12606.11557253071,
|
|
"request_id": 14,
|
|
"tpot_ms": 96.91984225378269,
|
|
"ttft_ms": 297.2956063003096
|
|
},
|
|
{
|
|
"e2e_ms": 12614.724705710416,
|
|
"request_id": 15,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 306.8357071486503
|
|
},
|
|
{
|
|
"e2e_ms": 12623.33383889012,
|
|
"request_id": 16,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 315.44484032835385
|
|
},
|
|
{
|
|
"e2e_ms": 12631.942972069823,
|
|
"request_id": 17,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 324.0539735080574
|
|
},
|
|
{
|
|
"e2e_ms": 12640.552105249526,
|
|
"request_id": 18,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 332.663106687761
|
|
},
|
|
{
|
|
"e2e_ms": 12649.16123842923,
|
|
"request_id": 19,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 341.2722398674646
|
|
},
|
|
{
|
|
"e2e_ms": 12657.770371608933,
|
|
"request_id": 20,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 349.88137304716815
|
|
},
|
|
{
|
|
"e2e_ms": 12666.379504788636,
|
|
"request_id": 21,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 358.49050622687173
|
|
},
|
|
{
|
|
"e2e_ms": 12674.988637968341,
|
|
"request_id": 22,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 367.0996394065753
|
|
},
|
|
{
|
|
"e2e_ms": 12683.597771148045,
|
|
"request_id": 23,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 375.7087725862789
|
|
},
|
|
{
|
|
"e2e_ms": 12692.945149660885,
|
|
"request_id": 24,
|
|
"tpot_ms": 96.91832475507798,
|
|
"ttft_ms": 384.31790576598246
|
|
},
|
|
{
|
|
"e2e_ms": 12606.580927732637,
|
|
"request_id": 25,
|
|
"tpot_ms": 96.91984225470249,
|
|
"ttft_ms": 297.7609613854213
|
|
},
|
|
{
|
|
"e2e_ms": 12615.190061213469,
|
|
"request_id": 26,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 307.3010623465393
|
|
},
|
|
{
|
|
"e2e_ms": 12623.7991946943,
|
|
"request_id": 27,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 315.9101958273709
|
|
},
|
|
{
|
|
"e2e_ms": 12632.408328175132,
|
|
"request_id": 28,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 324.5193293082025
|
|
},
|
|
{
|
|
"e2e_ms": 12641.017461655963,
|
|
"request_id": 29,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 333.1284627890341
|
|
},
|
|
{
|
|
"e2e_ms": 12649.626595136795,
|
|
"request_id": 30,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 341.73759626986566
|
|
},
|
|
{
|
|
"e2e_ms": 12658.235728617627,
|
|
"request_id": 31,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 350.34672975069725
|
|
},
|
|
{
|
|
"e2e_ms": 12666.844862098458,
|
|
"request_id": 32,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 358.95586323152884
|
|
},
|
|
{
|
|
"e2e_ms": 12675.45399557929,
|
|
"request_id": 33,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 367.5649967123604
|
|
},
|
|
{
|
|
"e2e_ms": 12684.063129060121,
|
|
"request_id": 34,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 376.174130193192
|
|
},
|
|
{
|
|
"e2e_ms": 12693.41050787034,
|
|
"request_id": 35,
|
|
"tpot_ms": 96.9183247574513,
|
|
"ttft_ms": 384.7832636740236
|
|
},
|
|
{
|
|
"e2e_ms": 12607.04628612126,
|
|
"request_id": 36,
|
|
"tpot_ms": 96.91984225613498,
|
|
"ttft_ms": 298.22631959211776
|
|
},
|
|
{
|
|
"e2e_ms": 12615.655419602092,
|
|
"request_id": 37,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 307.76642073516314
|
|
},
|
|
{
|
|
"e2e_ms": 12624.264553082923,
|
|
"request_id": 38,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 316.3755542159947
|
|
},
|
|
{
|
|
"e2e_ms": 12632.873686563755,
|
|
"request_id": 39,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 324.9846876968263
|
|
},
|
|
{
|
|
"e2e_ms": 12641.482820044586,
|
|
"request_id": 40,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 333.5938211776579
|
|
},
|
|
{
|
|
"e2e_ms": 12650.091953525418,
|
|
"request_id": 41,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 342.2029546584895
|
|
},
|
|
{
|
|
"e2e_ms": 12658.70108700625,
|
|
"request_id": 42,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 350.8120881393211
|
|
},
|
|
{
|
|
"e2e_ms": 12667.310220487081,
|
|
"request_id": 43,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 359.42122162015266
|
|
},
|
|
{
|
|
"e2e_ms": 12675.919353967913,
|
|
"request_id": 44,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 368.03035510098425
|
|
},
|
|
{
|
|
"e2e_ms": 12684.528487448744,
|
|
"request_id": 45,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 376.63948858181584
|
|
},
|
|
{
|
|
"e2e_ms": 12693.875866258964,
|
|
"request_id": 46,
|
|
"tpot_ms": 96.91832475745132,
|
|
"ttft_ms": 385.2486220626474
|
|
},
|
|
{
|
|
"e2e_ms": 12607.511644509885,
|
|
"request_id": 47,
|
|
"tpot_ms": 96.91984225613498,
|
|
"ttft_ms": 298.6916779807416
|
|
},
|
|
{
|
|
"e2e_ms": 12616.120777990716,
|
|
"request_id": 48,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 308.23177912378696
|
|
},
|
|
{
|
|
"e2e_ms": 12624.729911471548,
|
|
"request_id": 49,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 316.84091260461855
|
|
},
|
|
{
|
|
"e2e_ms": 12633.339044941862,
|
|
"request_id": 50,
|
|
"tpot_ms": 96.91251180201898,
|
|
"ttft_ms": 325.45004608545014
|
|
},
|
|
{
|
|
"e2e_ms": 12641.94817818975,
|
|
"request_id": 51,
|
|
"tpot_ms": 96.91251180018479,
|
|
"ttft_ms": 334.0591795662817
|
|
},
|
|
{
|
|
"e2e_ms": 12650.557311437638,
|
|
"request_id": 52,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 342.66831284185173
|
|
},
|
|
{
|
|
"e2e_ms": 12659.166444685525,
|
|
"request_id": 53,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 351.277446089739
|
|
},
|
|
{
|
|
"e2e_ms": 12667.775577933411,
|
|
"request_id": 54,
|
|
"tpot_ms": 96.91251179996681,
|
|
"ttft_ms": 359.88657933762624
|
|
},
|
|
{
|
|
"e2e_ms": 12676.3847111813,
|
|
"request_id": 55,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 368.4957125855135
|
|
},
|
|
{
|
|
"e2e_ms": 12684.993844429187,
|
|
"request_id": 56,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 377.10484583340076
|
|
},
|
|
{
|
|
"e2e_ms": 12694.3412230047,
|
|
"request_id": 57,
|
|
"tpot_ms": 96.91832475530246,
|
|
"ttft_ms": 385.713979081288
|
|
},
|
|
{
|
|
"e2e_ms": 12607.977001027848,
|
|
"request_id": 58,
|
|
"tpot_ms": 96.9198422540282,
|
|
"ttft_ms": 299.1570347662673
|
|
},
|
|
{
|
|
"e2e_ms": 12616.586134275734,
|
|
"request_id": 59,
|
|
"tpot_ms": 96.91251179996681,
|
|
"ttft_ms": 308.6971356799495
|
|
},
|
|
{
|
|
"e2e_ms": 12625.195267523623,
|
|
"request_id": 60,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 317.30626892783675
|
|
},
|
|
{
|
|
"e2e_ms": 12633.80440077151,
|
|
"request_id": 61,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 325.915402175724
|
|
},
|
|
{
|
|
"e2e_ms": 12642.413534019397,
|
|
"request_id": 62,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 334.52453542361127
|
|
},
|
|
{
|
|
"e2e_ms": 12436.81062947212,
|
|
"request_id": 63,
|
|
"tpot_ms": 95.22580284094978,
|
|
"ttft_ms": 343.1336686714985
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "37e166453683700c9df7abffa278431119205a29c83e92211ffa69d90e1dca0e"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 408.0712172985077,
|
|
"offered_request_rate": 0.2,
|
|
"request_metrics_sha256": "cf8a095cdf380d42316d1ad169ce9b2b45ae7d64348cd38271a332663de349ad",
|
|
"request_rate_per_gpu": 0.025,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12823.625057543633,
|
|
"request_id": 0,
|
|
"tpot_ms": 98.67683742367018,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 12920.533566322107,
|
|
"request_id": 1,
|
|
"tpot_ms": 99.10315669196875,
|
|
"ttft_ms": 334.4326664420745
|
|
},
|
|
{
|
|
"e2e_ms": 12921.724204826443,
|
|
"request_id": 2,
|
|
"tpot_ms": 99.22365133274592,
|
|
"ttft_ms": 320.32048556771286
|
|
},
|
|
{
|
|
"e2e_ms": 12922.91484333078,
|
|
"request_id": 3,
|
|
"tpot_ms": 99.22887081733072,
|
|
"ttft_ms": 320.84824952977976
|
|
},
|
|
{
|
|
"e2e_ms": 12924.105481838353,
|
|
"request_id": 4,
|
|
"tpot_ms": 99.22887081736695,
|
|
"ttft_ms": 322.0388880327505
|
|
},
|
|
{
|
|
"e2e_ms": 12925.296120352506,
|
|
"request_id": 5,
|
|
"tpot_ms": 99.22887081744425,
|
|
"ttft_ms": 323.2295265370873
|
|
},
|
|
{
|
|
"e2e_ms": 12926.486758866658,
|
|
"request_id": 6,
|
|
"tpot_ms": 99.22887081752154,
|
|
"ttft_ms": 324.420165041424
|
|
},
|
|
{
|
|
"e2e_ms": 12927.677397380812,
|
|
"request_id": 7,
|
|
"tpot_ms": 99.22887081749418,
|
|
"ttft_ms": 325.6108035590515
|
|
},
|
|
{
|
|
"e2e_ms": 12928.868035894964,
|
|
"request_id": 8,
|
|
"tpot_ms": 99.22887081749417,
|
|
"ttft_ms": 326.8014420732044
|
|
},
|
|
{
|
|
"e2e_ms": 12930.058674409118,
|
|
"request_id": 9,
|
|
"tpot_ms": 99.22887081749418,
|
|
"ttft_ms": 327.9920805873573
|
|
},
|
|
{
|
|
"e2e_ms": 12931.24931292327,
|
|
"request_id": 10,
|
|
"tpot_ms": 99.22887081749417,
|
|
"ttft_ms": 329.1827191015102
|
|
},
|
|
{
|
|
"e2e_ms": 12932.439951441467,
|
|
"request_id": 11,
|
|
"tpot_ms": 99.22887081752602,
|
|
"ttft_ms": 330.3733576156631
|
|
},
|
|
{
|
|
"e2e_ms": 12933.63058995827,
|
|
"request_id": 12,
|
|
"tpot_ms": 99.22887081754688,
|
|
"ttft_ms": 331.563996129816
|
|
},
|
|
{
|
|
"e2e_ms": 12934.821228475073,
|
|
"request_id": 13,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 332.7546346417307
|
|
},
|
|
{
|
|
"e2e_ms": 12936.011866991876,
|
|
"request_id": 14,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 333.94527315853395
|
|
},
|
|
{
|
|
"e2e_ms": 12937.20250550868,
|
|
"request_id": 15,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 335.1359116753372
|
|
},
|
|
{
|
|
"e2e_ms": 12938.393144025482,
|
|
"request_id": 16,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 336.3265501921404
|
|
},
|
|
{
|
|
"e2e_ms": 12939.583782542286,
|
|
"request_id": 17,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 337.51718870894365
|
|
},
|
|
{
|
|
"e2e_ms": 12940.77442105909,
|
|
"request_id": 18,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 338.7078272257469
|
|
},
|
|
{
|
|
"e2e_ms": 12941.965059575892,
|
|
"request_id": 19,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 339.8984657425501
|
|
},
|
|
{
|
|
"e2e_ms": 12943.155698092696,
|
|
"request_id": 20,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 341.08910425935335
|
|
},
|
|
{
|
|
"e2e_ms": 12944.346336609498,
|
|
"request_id": 21,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 342.2797427761566
|
|
},
|
|
{
|
|
"e2e_ms": 12945.536975126302,
|
|
"request_id": 22,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 343.4703812929598
|
|
},
|
|
{
|
|
"e2e_ms": 12946.727613643105,
|
|
"request_id": 23,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 344.66101980976305
|
|
},
|
|
{
|
|
"e2e_ms": 12947.918252142103,
|
|
"request_id": 24,
|
|
"tpot_ms": 99.22887081744517,
|
|
"ttft_ms": 345.8516583265663
|
|
},
|
|
{
|
|
"e2e_ms": 12949.10889064036,
|
|
"request_id": 25,
|
|
"tpot_ms": 99.22887081729912,
|
|
"ttft_ms": 347.0422968433695
|
|
},
|
|
{
|
|
"e2e_ms": 12950.299529138618,
|
|
"request_id": 26,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 348.23293534631716
|
|
},
|
|
{
|
|
"e2e_ms": 12951.490167636877,
|
|
"request_id": 27,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 349.42357384457523
|
|
},
|
|
{
|
|
"e2e_ms": 12952.680806135135,
|
|
"request_id": 28,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 350.6142123428333
|
|
},
|
|
{
|
|
"e2e_ms": 12953.871444633392,
|
|
"request_id": 29,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 351.80485084109137
|
|
},
|
|
{
|
|
"e2e_ms": 12955.06208313165,
|
|
"request_id": 30,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 352.99548933934943
|
|
},
|
|
{
|
|
"e2e_ms": 12956.25272162991,
|
|
"request_id": 31,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 354.1861278376075
|
|
},
|
|
{
|
|
"e2e_ms": 12957.443360128167,
|
|
"request_id": 32,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 355.37676633586557
|
|
},
|
|
{
|
|
"e2e_ms": 12958.633998626425,
|
|
"request_id": 33,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 356.56740483412364
|
|
},
|
|
{
|
|
"e2e_ms": 12959.824637124682,
|
|
"request_id": 34,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 357.7580433323817
|
|
},
|
|
{
|
|
"e2e_ms": 12961.015275622942,
|
|
"request_id": 35,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 358.94868183063977
|
|
},
|
|
{
|
|
"e2e_ms": 12962.2059141212,
|
|
"request_id": 36,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 360.13932032889784
|
|
},
|
|
{
|
|
"e2e_ms": 12963.396552619457,
|
|
"request_id": 37,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 361.3299588271559
|
|
},
|
|
{
|
|
"e2e_ms": 12964.587191117715,
|
|
"request_id": 38,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 362.520597325414
|
|
},
|
|
{
|
|
"e2e_ms": 12965.777829615974,
|
|
"request_id": 39,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 363.71123582367204
|
|
},
|
|
{
|
|
"e2e_ms": 12966.968468114232,
|
|
"request_id": 40,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 364.9018743219301
|
|
},
|
|
{
|
|
"e2e_ms": 12968.15910661249,
|
|
"request_id": 41,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 366.0925128201882
|
|
},
|
|
{
|
|
"e2e_ms": 12969.349745110747,
|
|
"request_id": 42,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 367.28315131844624
|
|
},
|
|
{
|
|
"e2e_ms": 12970.540383609006,
|
|
"request_id": 43,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 368.4737898167043
|
|
},
|
|
{
|
|
"e2e_ms": 12971.731022107264,
|
|
"request_id": 44,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 369.6644283149624
|
|
},
|
|
{
|
|
"e2e_ms": 12972.921660605522,
|
|
"request_id": 45,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 370.85506681322045
|
|
},
|
|
{
|
|
"e2e_ms": 12974.11229910378,
|
|
"request_id": 46,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 372.0457053114785
|
|
},
|
|
{
|
|
"e2e_ms": 12975.302937602039,
|
|
"request_id": 47,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 373.2363438097366
|
|
},
|
|
{
|
|
"e2e_ms": 12976.493576100296,
|
|
"request_id": 48,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 374.42698230799465
|
|
},
|
|
{
|
|
"e2e_ms": 12977.684214615067,
|
|
"request_id": 49,
|
|
"tpot_ms": 99.22887081739223,
|
|
"ttft_ms": 375.6176208062527
|
|
},
|
|
{
|
|
"e2e_ms": 12978.874853139188,
|
|
"request_id": 50,
|
|
"tpot_ms": 99.22887081759589,
|
|
"ttft_ms": 376.8082593045108
|
|
},
|
|
{
|
|
"e2e_ms": 12980.06549166331,
|
|
"request_id": 51,
|
|
"tpot_ms": 99.22887081779955,
|
|
"ttft_ms": 377.99889780276885
|
|
},
|
|
{
|
|
"e2e_ms": 12981.256130187432,
|
|
"request_id": 52,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 379.189536321519
|
|
},
|
|
{
|
|
"e2e_ms": 12982.446768711554,
|
|
"request_id": 53,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 380.3801748456408
|
|
},
|
|
{
|
|
"e2e_ms": 12983.637407235676,
|
|
"request_id": 54,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 381.5708133697626
|
|
},
|
|
{
|
|
"e2e_ms": 12984.828045759798,
|
|
"request_id": 55,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 382.76145189388444
|
|
},
|
|
{
|
|
"e2e_ms": 12986.01868428392,
|
|
"request_id": 56,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 383.95209041800626
|
|
},
|
|
{
|
|
"e2e_ms": 12987.209322808041,
|
|
"request_id": 57,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 385.1427289421281
|
|
},
|
|
{
|
|
"e2e_ms": 12988.399961332163,
|
|
"request_id": 58,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 386.3333674662499
|
|
},
|
|
{
|
|
"e2e_ms": 12989.881679406835,
|
|
"request_id": 59,
|
|
"tpot_ms": 99.2311627828068,
|
|
"ttft_ms": 387.52400599037173
|
|
},
|
|
{
|
|
"e2e_ms": 12991.363397481507,
|
|
"request_id": 60,
|
|
"tpot_ms": 99.23345474777176,
|
|
"ttft_ms": 388.71464451449356
|
|
},
|
|
{
|
|
"e2e_ms": 12896.836165730803,
|
|
"request_id": 61,
|
|
"tpot_ms": 99.23492287861664,
|
|
"ttft_ms": 294.0009601464908
|
|
},
|
|
{
|
|
"e2e_ms": 12689.81487892944,
|
|
"request_id": 62,
|
|
"tpot_ms": 97.59316693471084,
|
|
"ttft_ms": 295.4826782211626
|
|
},
|
|
{
|
|
"e2e_ms": 12448.118757066311,
|
|
"request_id": 63,
|
|
"tpot_ms": 95.67691266262345,
|
|
"ttft_ms": 297.1508489131338
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "e6cebfc858981b2ae4632d2c75f5e8c1f83280e7306351e15c7cd611d2664d80"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 220.7558696269989,
|
|
"offered_request_rate": 0.4,
|
|
"request_metrics_sha256": "e09d047b2039786ee02cd4c59c081ca95e5197ae65723b6170e43f6c0924921a",
|
|
"request_rate_per_gpu": 0.05,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 13556.392075191316,
|
|
"request_id": 0,
|
|
"tpot_ms": 104.4466564602661,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 13715.056219396885,
|
|
"request_id": 1,
|
|
"tpot_ms": 105.11756526634255,
|
|
"ttft_ms": 365.12543057138026
|
|
},
|
|
{
|
|
"e2e_ms": 13775.965021971426,
|
|
"request_id": 2,
|
|
"tpot_ms": 105.6336255570719,
|
|
"ttft_ms": 360.4945762232949
|
|
},
|
|
{
|
|
"e2e_ms": 13838.18765138338,
|
|
"request_id": 3,
|
|
"tpot_ms": 106.07242606904647,
|
|
"ttft_ms": 366.98954061447785
|
|
},
|
|
{
|
|
"e2e_ms": 13802.282961907104,
|
|
"request_id": 4,
|
|
"tpot_ms": 106.21712281117094,
|
|
"ttft_ms": 312.70836488839393
|
|
},
|
|
{
|
|
"e2e_ms": 13864.505591319055,
|
|
"request_id": 5,
|
|
"tpot_ms": 106.32495967758555,
|
|
"ttft_ms": 361.2357122656906
|
|
},
|
|
{
|
|
"e2e_ms": 13828.78708091747,
|
|
"request_id": 6,
|
|
"tpot_ms": 106.35892091285005,
|
|
"ttft_ms": 321.20412498551156
|
|
},
|
|
{
|
|
"e2e_ms": 13892.323156276025,
|
|
"request_id": 7,
|
|
"tpot_ms": 106.3810710779423,
|
|
"ttft_ms": 381.9271293773525
|
|
},
|
|
{
|
|
"e2e_ms": 13856.60464587857,
|
|
"request_id": 8,
|
|
"tpot_ms": 106.3914161711828,
|
|
"ttft_ms": 344.89479213835494
|
|
},
|
|
{
|
|
"e2e_ms": 13820.886135481267,
|
|
"request_id": 9,
|
|
"tpot_ms": 106.39288214818258,
|
|
"ttft_ms": 308.9901026620794
|
|
},
|
|
{
|
|
"e2e_ms": 13883.108764897528,
|
|
"request_id": 10,
|
|
"tpot_ms": 106.39288214821651,
|
|
"ttft_ms": 371.21273207403146
|
|
},
|
|
{
|
|
"e2e_ms": 13847.390254500226,
|
|
"request_id": 11,
|
|
"tpot_ms": 106.39288214825024,
|
|
"ttft_ms": 335.4942216724446
|
|
},
|
|
{
|
|
"e2e_ms": 13811.485565028235,
|
|
"request_id": 12,
|
|
"tpot_ms": 106.39288214828399,
|
|
"ttft_ms": 299.589532196169
|
|
},
|
|
{
|
|
"e2e_ms": 13873.708194444496,
|
|
"request_id": 13,
|
|
"tpot_ms": 106.38107407728566,
|
|
"ttft_ms": 363.31178662921815
|
|
},
|
|
{
|
|
"e2e_ms": 13837.989684047194,
|
|
"request_id": 14,
|
|
"tpot_ms": 106.38107407728566,
|
|
"ttft_ms": 327.5932762319158
|
|
},
|
|
{
|
|
"e2e_ms": 13901.52575941027,
|
|
"request_id": 15,
|
|
"tpot_ms": 106.39141617135505,
|
|
"ttft_ms": 389.81590564817736
|
|
},
|
|
{
|
|
"e2e_ms": 13865.807249012967,
|
|
"request_id": 16,
|
|
"tpot_ms": 106.39141617135505,
|
|
"ttft_ms": 354.097395250875
|
|
},
|
|
{
|
|
"e2e_ms": 13830.088738615665,
|
|
"request_id": 17,
|
|
"tpot_ms": 106.3928821483211,
|
|
"ttft_ms": 318.19270577888403
|
|
},
|
|
{
|
|
"e2e_ms": 13893.62481397874,
|
|
"request_id": 18,
|
|
"tpot_ms": 106.40322424239051,
|
|
"ttft_ms": 380.4153351951456
|
|
},
|
|
{
|
|
"e2e_ms": 13857.906303581438,
|
|
"request_id": 19,
|
|
"tpot_ms": 106.40322424239051,
|
|
"ttft_ms": 344.69682479784325
|
|
},
|
|
{
|
|
"e2e_ms": 13822.187793184135,
|
|
"request_id": 20,
|
|
"tpot_ms": 106.40469021935657,
|
|
"ttft_ms": 308.79213532585226
|
|
},
|
|
{
|
|
"e2e_ms": 13884.410422593844,
|
|
"request_id": 21,
|
|
"tpot_ms": 106.39288214826952,
|
|
"ttft_ms": 372.51438976361584
|
|
},
|
|
{
|
|
"e2e_ms": 13848.691912190276,
|
|
"request_id": 22,
|
|
"tpot_ms": 106.39288214822018,
|
|
"ttft_ms": 336.7958793663135
|
|
},
|
|
{
|
|
"e2e_ms": 13812.78722271334,
|
|
"request_id": 23,
|
|
"tpot_ms": 106.39288214818123,
|
|
"ttft_ms": 300.8911898943225
|
|
},
|
|
{
|
|
"e2e_ms": 13875.009852123696,
|
|
"request_id": 24,
|
|
"tpot_ms": 106.38107407709929,
|
|
"ttft_ms": 364.6134443320861
|
|
},
|
|
{
|
|
"e2e_ms": 13839.291341720127,
|
|
"request_id": 25,
|
|
"tpot_ms": 106.38107407704995,
|
|
"ttft_ms": 328.89493393478375
|
|
},
|
|
{
|
|
"e2e_ms": 13902.827417078257,
|
|
"request_id": 26,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 391.11756334850156
|
|
},
|
|
{
|
|
"e2e_ms": 13867.108906674688,
|
|
"request_id": 27,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 355.39905294493224
|
|
},
|
|
{
|
|
"e2e_ms": 13831.390396271117,
|
|
"request_id": 28,
|
|
"tpot_ms": 106.39288214805607,
|
|
"ttft_ms": 319.49436346799587
|
|
},
|
|
{
|
|
"e2e_ms": 13894.926471629247,
|
|
"request_id": 29,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 381.7169928783528
|
|
},
|
|
{
|
|
"e2e_ms": 13859.207961225678,
|
|
"request_id": 30,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 345.9984824747835
|
|
},
|
|
{
|
|
"e2e_ms": 13823.48945082211,
|
|
"request_id": 31,
|
|
"tpot_ms": 106.40469021908869,
|
|
"ttft_ms": 310.0937929978471
|
|
},
|
|
{
|
|
"e2e_ms": 13885.712080232466,
|
|
"request_id": 32,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 373.8160474293437
|
|
},
|
|
{
|
|
"e2e_ms": 13849.993569828897,
|
|
"request_id": 33,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 338.0975370257744
|
|
},
|
|
{
|
|
"e2e_ms": 13814.08888035196,
|
|
"request_id": 34,
|
|
"tpot_ms": 106.39288214805607,
|
|
"ttft_ms": 302.192847548838
|
|
},
|
|
{
|
|
"e2e_ms": 13876.311509762318,
|
|
"request_id": 35,
|
|
"tpot_ms": 106.3810740770235,
|
|
"ttft_ms": 365.9151019803346
|
|
},
|
|
{
|
|
"e2e_ms": 13840.592999358747,
|
|
"request_id": 36,
|
|
"tpot_ms": 106.38107407702348,
|
|
"ttft_ms": 330.1965915767653
|
|
},
|
|
{
|
|
"e2e_ms": 13904.129074716877,
|
|
"request_id": 37,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 392.41922098712223
|
|
},
|
|
{
|
|
"e2e_ms": 13868.410564313308,
|
|
"request_id": 38,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 356.7007105835529
|
|
},
|
|
{
|
|
"e2e_ms": 13832.69205390974,
|
|
"request_id": 39,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 320.79602110661654
|
|
},
|
|
{
|
|
"e2e_ms": 13896.228129267869,
|
|
"request_id": 40,
|
|
"tpot_ms": 106.40322424213304,
|
|
"ttft_ms": 383.0186505169735
|
|
},
|
|
{
|
|
"e2e_ms": 13860.509618864298,
|
|
"request_id": 41,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 347.30014011340415
|
|
},
|
|
{
|
|
"e2e_ms": 13824.79110846073,
|
|
"request_id": 42,
|
|
"tpot_ms": 106.40469021908868,
|
|
"ttft_ms": 311.3954506364678
|
|
},
|
|
{
|
|
"e2e_ms": 13888.32718381886,
|
|
"request_id": 43,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 375.1177050679644
|
|
},
|
|
{
|
|
"e2e_ms": 13852.60867341529,
|
|
"request_id": 44,
|
|
"tpot_ms": 106.40322424213304,
|
|
"ttft_ms": 339.39919466439505
|
|
},
|
|
{
|
|
"e2e_ms": 13816.89016301172,
|
|
"request_id": 45,
|
|
"tpot_ms": 106.40469021908868,
|
|
"ttft_ms": 303.4945051874587
|
|
},
|
|
{
|
|
"e2e_ms": 13879.11279242067,
|
|
"request_id": 46,
|
|
"tpot_ms": 106.392882148045,
|
|
"ttft_ms": 367.2167596189553
|
|
},
|
|
{
|
|
"e2e_ms": 13843.394282013549,
|
|
"request_id": 47,
|
|
"tpot_ms": 106.39288214801702,
|
|
"ttft_ms": 331.49824921538595
|
|
},
|
|
{
|
|
"e2e_ms": 13807.489592530403,
|
|
"request_id": 48,
|
|
"tpot_ms": 106.39288214796812,
|
|
"ttft_ms": 295.5935597384496
|
|
},
|
|
{
|
|
"e2e_ms": 13869.712221935884,
|
|
"request_id": 49,
|
|
"tpot_ms": 106.38107407689715,
|
|
"ttft_ms": 359.3158141699462
|
|
},
|
|
{
|
|
"e2e_ms": 13833.993711528763,
|
|
"request_id": 50,
|
|
"tpot_ms": 106.38107407686918,
|
|
"ttft_ms": 323.59730376637685
|
|
},
|
|
{
|
|
"e2e_ms": 13897.5297868798,
|
|
"request_id": 51,
|
|
"tpot_ms": 106.3914161708903,
|
|
"ttft_ms": 385.8199331767338
|
|
},
|
|
{
|
|
"e2e_ms": 13861.81127647268,
|
|
"request_id": 52,
|
|
"tpot_ms": 106.39141617089378,
|
|
"ttft_ms": 350.1014227691712
|
|
},
|
|
{
|
|
"e2e_ms": 13826.092766065556,
|
|
"request_id": 53,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 314.1967332860247
|
|
},
|
|
{
|
|
"e2e_ms": 13889.628841416596,
|
|
"request_id": 54,
|
|
"tpot_ms": 106.40322424192983,
|
|
"ttft_ms": 376.41936269150733
|
|
},
|
|
{
|
|
"e2e_ms": 13853.910331009472,
|
|
"request_id": 55,
|
|
"tpot_ms": 106.4032242419298,
|
|
"ttft_ms": 340.7008522843853
|
|
},
|
|
{
|
|
"e2e_ms": 13818.191820602351,
|
|
"request_id": 56,
|
|
"tpot_ms": 106.4046902189064,
|
|
"ttft_ms": 304.7961628012388
|
|
},
|
|
{
|
|
"e2e_ms": 13880.414450007833,
|
|
"request_id": 57,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 368.5184172283016
|
|
},
|
|
{
|
|
"e2e_ms": 13844.695939600711,
|
|
"request_id": 58,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 332.79990682117955
|
|
},
|
|
{
|
|
"e2e_ms": 13600.134841430987,
|
|
"request_id": 59,
|
|
"tpot_ms": 104.74991829994453,
|
|
"ttft_ms": 296.89521733803304
|
|
},
|
|
{
|
|
"e2e_ms": 13451.749583806417,
|
|
"request_id": 60,
|
|
"tpot_ms": 103.07978040977419,
|
|
"ttft_ms": 360.61747176509584
|
|
},
|
|
{
|
|
"e2e_ms": 13168.506714577745,
|
|
"request_id": 61,
|
|
"tpot_ms": 101.13076971039189,
|
|
"ttft_ms": 324.8989613579738
|
|
},
|
|
{
|
|
"e2e_ms": 12970.21046401943,
|
|
"request_id": 62,
|
|
"tpot_ms": 99.07943994689744,
|
|
"ttft_ms": 387.1215907634564
|
|
},
|
|
{
|
|
"e2e_ms": 12659.030247905519,
|
|
"request_id": 63,
|
|
"tpot_ms": 96.91045013818255,
|
|
"ttft_ms": 351.4030803563344
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "021442528b6c4067d63827496b66b6f530adbe9f4b29732b1aa7aab2112ced9f"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 142.3273642063141,
|
|
"offered_request_rate": 0.8,
|
|
"request_metrics_sha256": "6b246d56546536e46b4e8adfb23076b3cbf053da4b0af0288346df10122d6022",
|
|
"request_rate_per_gpu": 0.1,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 15423.90313583537,
|
|
"request_id": 0,
|
|
"tpot_ms": 119.15146796140039,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 15680.782114004433,
|
|
"request_id": 1,
|
|
"tpot_ms": 120.47488426274414,
|
|
"ttft_ms": 380.47181263592745
|
|
},
|
|
{
|
|
"e2e_ms": 15824.99160915063,
|
|
"request_id": 2,
|
|
"tpot_ms": 121.60222806639713,
|
|
"ttft_ms": 381.5086447181937
|
|
},
|
|
{
|
|
"e2e_ms": 15964.925902693089,
|
|
"request_id": 3,
|
|
"tpot_ms": 122.6571852387084,
|
|
"ttft_ms": 387.4633773771219
|
|
},
|
|
{
|
|
"e2e_ms": 15994.3969757696,
|
|
"request_id": 4,
|
|
"tpot_ms": 123.48492728290344,
|
|
"ttft_ms": 311.8112108408626
|
|
},
|
|
{
|
|
"e2e_ms": 16132.273068839513,
|
|
"request_id": 5,
|
|
"tpot_ms": 124.38017511477935,
|
|
"ttft_ms": 335.9908292625358
|
|
},
|
|
{
|
|
"e2e_ms": 16268.09096143688,
|
|
"request_id": 6,
|
|
"tpot_ms": 125.15372351796954,
|
|
"ttft_ms": 373.5680746547478
|
|
},
|
|
{
|
|
"e2e_ms": 16291.239371010352,
|
|
"request_id": 7,
|
|
"tpot_ms": 125.67763083904141,
|
|
"ttft_ms": 330.18025445209173
|
|
},
|
|
{
|
|
"e2e_ms": 16612.993923996273,
|
|
"request_id": 8,
|
|
"tpot_ms": 127.71634872045071,
|
|
"ttft_ms": 393.0176364990334
|
|
},
|
|
{
|
|
"e2e_ms": 16636.356874836954,
|
|
"request_id": 9,
|
|
"tpot_ms": 128.11558123152213,
|
|
"ttft_ms": 365.67805843364454
|
|
},
|
|
{
|
|
"e2e_ms": 16659.624174048608,
|
|
"request_id": 10,
|
|
"tpot_ms": 128.3236060799542,
|
|
"ttft_ms": 362.52620189442376
|
|
},
|
|
{
|
|
"e2e_ms": 16683.03953534565,
|
|
"request_id": 11,
|
|
"tpot_ms": 128.34365422438844,
|
|
"ttft_ms": 383.3954488483169
|
|
},
|
|
{
|
|
"e2e_ms": 16598.14552827833,
|
|
"request_id": 12,
|
|
"tpot_ms": 128.20865630197483,
|
|
"ttft_ms": 315.6461779275297
|
|
},
|
|
{
|
|
"e2e_ms": 16621.508479120537,
|
|
"request_id": 13,
|
|
"tpot_ms": 128.0747814081899,
|
|
"ttft_ms": 356.0112402804201
|
|
},
|
|
{
|
|
"e2e_ms": 16644.871429963147,
|
|
"request_id": 14,
|
|
"tpot_ms": 127.97565080664988,
|
|
"ttft_ms": 391.9637775186118
|
|
},
|
|
{
|
|
"e2e_ms": 16559.977422897646,
|
|
"request_id": 15,
|
|
"tpot_ms": 127.91134905305496,
|
|
"ttft_ms": 315.2360931596654
|
|
},
|
|
{
|
|
"e2e_ms": 16583.340373740255,
|
|
"request_id": 16,
|
|
"tpot_ms": 127.86325360239432,
|
|
"ttft_ms": 344.7071662361765
|
|
},
|
|
{
|
|
"e2e_ms": 16606.703324582868,
|
|
"request_id": 17,
|
|
"tpot_ms": 127.81515815173371,
|
|
"ttft_ms": 374.17823931268754
|
|
},
|
|
{
|
|
"e2e_ms": 16630.066275425477,
|
|
"request_id": 18,
|
|
"tpot_ms": 127.7832690040065,
|
|
"ttft_ms": 401.59111191665176
|
|
},
|
|
{
|
|
"e2e_ms": 16354.971144941388,
|
|
"request_id": 19,
|
|
"tpot_ms": 126.2873116642462,
|
|
"ttft_ms": 316.48256358212024
|
|
},
|
|
{
|
|
"e2e_ms": 16568.46874002116,
|
|
"request_id": 20,
|
|
"tpot_ms": 127.78612414854776,
|
|
"ttft_ms": 339.6309731555931
|
|
},
|
|
{
|
|
"e2e_ms": 16591.83169086377,
|
|
"request_id": 21,
|
|
"tpot_ms": 127.78612414856296,
|
|
"ttft_ms": 362.9939239962745
|
|
},
|
|
{
|
|
"e2e_ms": 16615.19464170638,
|
|
"request_id": 22,
|
|
"tpot_ms": 127.78612414857815,
|
|
"ttft_ms": 386.3568748369559
|
|
},
|
|
{
|
|
"e2e_ms": 16339.951449136564,
|
|
"request_id": 23,
|
|
"tpot_ms": 126.28847750749865,
|
|
"ttft_ms": 301.3148056842354
|
|
},
|
|
{
|
|
"e2e_ms": 16553.449044216337,
|
|
"request_id": 24,
|
|
"tpot_ms": 127.78518800972483,
|
|
"ttft_ms": 324.73016698128276
|
|
},
|
|
{
|
|
"e2e_ms": 16576.811995058946,
|
|
"request_id": 25,
|
|
"tpot_ms": 127.78477532898123,
|
|
"ttft_ms": 348.1455282783301
|
|
},
|
|
{
|
|
"e2e_ms": 16600.174945901555,
|
|
"request_id": 26,
|
|
"tpot_ms": 127.78477532898441,
|
|
"ttft_ms": 371.50847912053564
|
|
},
|
|
{
|
|
"e2e_ms": 16623.53789674417,
|
|
"request_id": 27,
|
|
"tpot_ms": 127.78477532898442,
|
|
"ttft_ms": 394.87142996314617
|
|
},
|
|
{
|
|
"e2e_ms": 16348.29470417435,
|
|
"request_id": 28,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 309.97742289764574
|
|
},
|
|
{
|
|
"e2e_ms": 16561.79229925412,
|
|
"request_id": 29,
|
|
"tpot_ms": 127.78308602766822,
|
|
"ttft_ms": 333.34037374025627
|
|
},
|
|
{
|
|
"e2e_ms": 16585.155250096734,
|
|
"request_id": 30,
|
|
"tpot_ms": 127.78308602766825,
|
|
"ttft_ms": 356.7033245828668
|
|
},
|
|
{
|
|
"e2e_ms": 16608.518200939343,
|
|
"request_id": 31,
|
|
"tpot_ms": 127.78308602766823,
|
|
"ttft_ms": 380.06627542547733
|
|
},
|
|
{
|
|
"e2e_ms": 16631.881151781956,
|
|
"request_id": 32,
|
|
"tpot_ms": 127.78360948578973,
|
|
"ttft_ms": 403.36274708666053
|
|
},
|
|
{
|
|
"e2e_ms": 16356.786021297865,
|
|
"request_id": 33,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 318.4687400211601
|
|
},
|
|
{
|
|
"e2e_ms": 16570.283616377637,
|
|
"request_id": 34,
|
|
"tpot_ms": 127.78308602766825,
|
|
"ttft_ms": 341.83169086377063
|
|
},
|
|
{
|
|
"e2e_ms": 16593.646567220247,
|
|
"request_id": 35,
|
|
"tpot_ms": 127.78308602766823,
|
|
"ttft_ms": 365.19464170638116
|
|
},
|
|
{
|
|
"e2e_ms": 16617.00951806286,
|
|
"request_id": 36,
|
|
"tpot_ms": 127.78477532898444,
|
|
"ttft_ms": 388.34305128183644
|
|
},
|
|
{
|
|
"e2e_ms": 16341.766325493041,
|
|
"request_id": 37,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 303.449044216336
|
|
},
|
|
{
|
|
"e2e_ms": 16555.263920572714,
|
|
"request_id": 38,
|
|
"tpot_ms": 127.78308602766747,
|
|
"ttft_ms": 326.81199505894654
|
|
},
|
|
{
|
|
"e2e_ms": 16578.62687141207,
|
|
"request_id": 39,
|
|
"tpot_ms": 127.78308602764184,
|
|
"ttft_ms": 350.17494590155707
|
|
},
|
|
{
|
|
"e2e_ms": 16601.989822251424,
|
|
"request_id": 40,
|
|
"tpot_ms": 127.78308602761618,
|
|
"ttft_ms": 373.5378967441676
|
|
},
|
|
{
|
|
"e2e_ms": 16625.35277309078,
|
|
"request_id": 41,
|
|
"tpot_ms": 127.78477532890675,
|
|
"ttft_ms": 396.6863063196229
|
|
},
|
|
{
|
|
"e2e_ms": 16350.109580518321,
|
|
"request_id": 42,
|
|
"tpot_ms": 126.2859628446,
|
|
"ttft_ms": 311.79229925412244
|
|
},
|
|
{
|
|
"e2e_ms": 16563.607175594883,
|
|
"request_id": 43,
|
|
"tpot_ms": 127.78308602754448,
|
|
"ttft_ms": 335.15525009673297
|
|
},
|
|
{
|
|
"e2e_ms": 16586.970126434237,
|
|
"request_id": 44,
|
|
"tpot_ms": 127.78308602751885,
|
|
"ttft_ms": 358.5182009393435
|
|
},
|
|
{
|
|
"e2e_ms": 16610.333077273594,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.78308602749323,
|
|
"ttft_ms": 381.88115178195403
|
|
},
|
|
{
|
|
"e2e_ms": 16633.69602811295,
|
|
"request_id": 46,
|
|
"tpot_ms": 127.78360948558907,
|
|
"ttft_ms": 405.17762344313724
|
|
},
|
|
{
|
|
"e2e_ms": 16548.802021044266,
|
|
"request_id": 47,
|
|
"tpot_ms": 127.78360948556401,
|
|
"ttft_ms": 320.2836163776368
|
|
},
|
|
{
|
|
"e2e_ms": 16572.164971883623,
|
|
"request_id": 48,
|
|
"tpot_ms": 127.7836094855384,
|
|
"ttft_ms": 343.64656722024733
|
|
},
|
|
{
|
|
"e2e_ms": 16595.52792272298,
|
|
"request_id": 49,
|
|
"tpot_ms": 127.78360948551277,
|
|
"ttft_ms": 367.00951806285786
|
|
},
|
|
{
|
|
"e2e_ms": 16618.890873562337,
|
|
"request_id": 50,
|
|
"tpot_ms": 127.78529878680334,
|
|
"ttft_ms": 390.15792763831314
|
|
},
|
|
{
|
|
"e2e_ms": 16343.647680989874,
|
|
"request_id": 51,
|
|
"tpot_ms": 126.28648630249732,
|
|
"ttft_ms": 305.26392057271323
|
|
},
|
|
{
|
|
"e2e_ms": 16199.563507361632,
|
|
"request_id": 52,
|
|
"tpot_ms": 124.96800500747688,
|
|
"ttft_ms": 328.6268714120695
|
|
},
|
|
{
|
|
"e2e_ms": 16032.332947106512,
|
|
"request_id": 53,
|
|
"tpot_ms": 123.46726869964634,
|
|
"ttft_ms": 351.9898222514257
|
|
},
|
|
{
|
|
"e2e_ms": 15838.885472588829,
|
|
"request_id": 54,
|
|
"tpot_ms": 121.7600999960476,
|
|
"ttft_ms": 375.35277309078197
|
|
},
|
|
{
|
|
"e2e_ms": 15619.209002299955,
|
|
"request_id": 55,
|
|
"tpot_ms": 119.84809306800307,
|
|
"ttft_ms": 398.5011826635656
|
|
},
|
|
{
|
|
"e2e_ms": 15285.473398871076,
|
|
"request_id": 56,
|
|
"tpot_ms": 117.8887104194976,
|
|
"ttft_ms": 313.60717559488194
|
|
},
|
|
{
|
|
"e2e_ms": 15049.070462905931,
|
|
"request_id": 57,
|
|
"tpot_ms": 115.84330973599758,
|
|
"ttft_ms": 336.9701264342382
|
|
},
|
|
{
|
|
"e2e_ms": 14793.47811125659,
|
|
"request_id": 58,
|
|
"tpot_ms": 113.6468112912047,
|
|
"ttft_ms": 360.3330772735944
|
|
},
|
|
{
|
|
"e2e_ms": 14524.751300129523,
|
|
"request_id": 59,
|
|
"tpot_ms": 111.34689190564231,
|
|
"ttft_ms": 383.69602811295067
|
|
},
|
|
{
|
|
"e2e_ms": 14156.606828520311,
|
|
"request_id": 60,
|
|
"tpot_ms": 109.11657328721294,
|
|
"ttft_ms": 298.802021044267
|
|
},
|
|
{
|
|
"e2e_ms": 13870.414276689999,
|
|
"request_id": 61,
|
|
"tpot_ms": 106.67912838430216,
|
|
"ttft_ms": 322.16497188362325
|
|
},
|
|
{
|
|
"e2e_ms": 13579.457505623892,
|
|
"request_id": 62,
|
|
"tpot_ms": 104.20416994410166,
|
|
"ttft_ms": 345.5279227229795
|
|
},
|
|
{
|
|
"e2e_ms": 13281.118281252404,
|
|
"request_id": 63,
|
|
"tpot_ms": 101.6710819503155,
|
|
"ttft_ms": 368.89087356233574
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.15625,
|
|
"passed": 10
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "9ed2bf69d3193253248833822a357db5130e6ca29282e93e04cc92961dba468d"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 138.39458417892456,
|
|
"offered_request_rate": 1.2,
|
|
"request_metrics_sha256": "dfafc592cb8f7c26725458265ca04cb89642135218d5f80993dfbd6e83fcea9c",
|
|
"request_rate_per_gpu": 0.15,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 17349.6420929535,
|
|
"request_id": 0,
|
|
"tpot_ms": 134.3147668363463,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 17472.38469413562,
|
|
"request_id": 1,
|
|
"tpot_ms": 135.04025977135157,
|
|
"ttft_ms": 322.27170317397037
|
|
},
|
|
{
|
|
"e2e_ms": 17594.570506858923,
|
|
"request_id": 2,
|
|
"tpot_ms": 135.72808492940857,
|
|
"ttft_ms": 357.1037208240346
|
|
},
|
|
{
|
|
"e2e_ms": 17607.2367009099,
|
|
"request_id": 3,
|
|
"tpot_ms": 136.28731175562888,
|
|
"ttft_ms": 298.7481079450336
|
|
},
|
|
{
|
|
"e2e_ms": 17921.914449676293,
|
|
"request_id": 4,
|
|
"tpot_ms": 138.39011926907332,
|
|
"ttft_ms": 346.36930250398024
|
|
},
|
|
{
|
|
"e2e_ms": 17935.406822135243,
|
|
"request_id": 5,
|
|
"tpot_ms": 138.88107821741167,
|
|
"ttft_ms": 297.50988852396
|
|
},
|
|
{
|
|
"e2e_ms": 18057.925904932094,
|
|
"request_id": 6,
|
|
"tpot_ms": 139.39521236971183,
|
|
"ttft_ms": 354.73393397869125
|
|
},
|
|
{
|
|
"e2e_ms": 18071.482157513437,
|
|
"request_id": 7,
|
|
"tpot_ms": 139.75067177492934,
|
|
"ttft_ms": 323.14684209741085
|
|
},
|
|
{
|
|
"e2e_ms": 18194.065120430685,
|
|
"request_id": 8,
|
|
"tpot_ms": 140.142283173617,
|
|
"ttft_ms": 395.99515738132584
|
|
},
|
|
{
|
|
"e2e_ms": 18207.834601589766,
|
|
"request_id": 9,
|
|
"tpot_ms": 140.4257392143267,
|
|
"ttft_ms": 373.76572137027256
|
|
},
|
|
{
|
|
"e2e_ms": 18221.607066695313,
|
|
"request_id": 10,
|
|
"tpot_ms": 140.60137526077196,
|
|
"ttft_ms": 365.232408577274
|
|
},
|
|
{
|
|
"e2e_ms": 18235.379531799863,
|
|
"request_id": 11,
|
|
"tpot_ms": 140.670460527253,
|
|
"ttft_ms": 370.23104483873226
|
|
},
|
|
{
|
|
"e2e_ms": 18249.21289308034,
|
|
"request_id": 12,
|
|
"tpot_ms": 140.64257855984852,
|
|
"ttft_ms": 387.6054159795803
|
|
},
|
|
{
|
|
"e2e_ms": 18263.046254360816,
|
|
"request_id": 13,
|
|
"tpot_ms": 140.69995603545655,
|
|
"ttft_ms": 394.1518378578337
|
|
},
|
|
{
|
|
"e2e_ms": 18276.879615640297,
|
|
"request_id": 14,
|
|
"tpot_ms": 140.75508408264292,
|
|
"ttft_ms": 400.98393714464817
|
|
},
|
|
{
|
|
"e2e_ms": 18181.536918127527,
|
|
"request_id": 15,
|
|
"tpot_ms": 140.80327015717634,
|
|
"ttft_ms": 299.52160816613116
|
|
},
|
|
{
|
|
"e2e_ms": 18195.15705083027,
|
|
"request_id": 16,
|
|
"tpot_ms": 140.84624138271937,
|
|
"ttft_ms": 307.6843952249071
|
|
},
|
|
{
|
|
"e2e_ms": 18208.77718353195,
|
|
"request_id": 17,
|
|
"tpot_ms": 140.87800086343316,
|
|
"ttft_ms": 317.2710738759417
|
|
},
|
|
{
|
|
"e2e_ms": 18222.54666469151,
|
|
"request_id": 18,
|
|
"tpot_ms": 140.90607860065492,
|
|
"ttft_ms": 327.4746824083348
|
|
},
|
|
{
|
|
"e2e_ms": 18236.319129797193,
|
|
"request_id": 19,
|
|
"tpot_ms": 140.91088449209656,
|
|
"ttft_ms": 340.6367993009294
|
|
},
|
|
{
|
|
"e2e_ms": 18250.152491077173,
|
|
"request_id": 20,
|
|
"tpot_ms": 140.8981539798204,
|
|
"ttft_ms": 356.0869356399827
|
|
},
|
|
{
|
|
"e2e_ms": 18263.985852358146,
|
|
"request_id": 21,
|
|
"tpot_ms": 140.89857595079965,
|
|
"ttft_ms": 369.8667066065901
|
|
},
|
|
{
|
|
"e2e_ms": 18277.81921363912,
|
|
"request_id": 22,
|
|
"tpot_ms": 140.90338208286406,
|
|
"ttft_ms": 383.0896891153834
|
|
},
|
|
{
|
|
"e2e_ms": 18291.6525749191,
|
|
"request_id": 23,
|
|
"tpot_ms": 140.91257237601366,
|
|
"ttft_ms": 395.7558831653678
|
|
},
|
|
{
|
|
"e2e_ms": 18305.48593620007,
|
|
"request_id": 24,
|
|
"tpot_ms": 140.9217626691632,
|
|
"ttft_ms": 408.4220772163469
|
|
},
|
|
{
|
|
"e2e_ms": 18210.143238687502,
|
|
"request_id": 25,
|
|
"tpot_ms": 140.92327164841032,
|
|
"ttft_ms": 312.8877393393914
|
|
},
|
|
{
|
|
"e2e_ms": 18223.912719846063,
|
|
"request_id": 26,
|
|
"tpot_ms": 140.92545360667498,
|
|
"ttft_ms": 326.38011179834336
|
|
},
|
|
{
|
|
"e2e_ms": 18237.68518495175,
|
|
"request_id": 27,
|
|
"tpot_ms": 140.92765906057843,
|
|
"ttft_ms": 339.8724842582901
|
|
},
|
|
{
|
|
"e2e_ms": 18251.518546232724,
|
|
"request_id": 28,
|
|
"tpot_ms": 140.92984101884323,
|
|
"ttft_ms": 353.42873683963205
|
|
},
|
|
{
|
|
"e2e_ms": 18265.3519075127,
|
|
"request_id": 29,
|
|
"tpot_ms": 140.93202297710803,
|
|
"ttft_ms": 366.98498941997923
|
|
},
|
|
{
|
|
"e2e_ms": 18279.185268793677,
|
|
"request_id": 30,
|
|
"tpot_ms": 140.93370194229104,
|
|
"ttft_ms": 380.60512212271647
|
|
},
|
|
{
|
|
"e2e_ms": 18293.01863007465,
|
|
"request_id": 31,
|
|
"tpot_ms": 140.9341814397353,
|
|
"ttft_ms": 394.3775872282629
|
|
},
|
|
{
|
|
"e2e_ms": 18306.851991354626,
|
|
"request_id": 32,
|
|
"tpot_ms": 140.93466093717961,
|
|
"ttft_ms": 408.15005233281454
|
|
},
|
|
{
|
|
"e2e_ms": 18211.509293842057,
|
|
"request_id": 33,
|
|
"tpot_ms": 140.933484965084,
|
|
"ttft_ms": 312.956703276388
|
|
},
|
|
{
|
|
"e2e_ms": 18225.278775001614,
|
|
"request_id": 34,
|
|
"tpot_ms": 140.9329819720059,
|
|
"ttft_ms": 326.79006455686374
|
|
},
|
|
{
|
|
"e2e_ms": 18239.11213628159,
|
|
"request_id": 35,
|
|
"tpot_ms": 140.93298197200983,
|
|
"ttft_ms": 340.6234258363447
|
|
},
|
|
{
|
|
"e2e_ms": 18252.945497562563,
|
|
"request_id": 36,
|
|
"tpot_ms": 140.93298197201372,
|
|
"ttft_ms": 354.45678711682046
|
|
},
|
|
{
|
|
"e2e_ms": 18266.778858843536,
|
|
"request_id": 37,
|
|
"tpot_ms": 140.93466093719667,
|
|
"ttft_ms": 368.0769198195577
|
|
},
|
|
{
|
|
"e2e_ms": 18280.612220123516,
|
|
"request_id": 38,
|
|
"tpot_ms": 140.93633990237933,
|
|
"ttft_ms": 381.69705252133923
|
|
},
|
|
{
|
|
"e2e_ms": 18294.44558140449,
|
|
"request_id": 39,
|
|
"tpot_ms": 140.93801886756114,
|
|
"ttft_ms": 395.3171852242221
|
|
},
|
|
{
|
|
"e2e_ms": 18308.27894268546,
|
|
"request_id": 40,
|
|
"tpot_ms": 140.93849836500434,
|
|
"ttft_ms": 409.0896503299106
|
|
},
|
|
{
|
|
"e2e_ms": 18212.936245171895,
|
|
"request_id": 41,
|
|
"tpot_ms": 140.93732239290452,
|
|
"ttft_ms": 313.8963012730187
|
|
},
|
|
{
|
|
"e2e_ms": 18226.708710277584,
|
|
"request_id": 42,
|
|
"tpot_ms": 140.93684289546135,
|
|
"ttft_ms": 327.72966255399183
|
|
},
|
|
{
|
|
"e2e_ms": 18047.749752064,
|
|
"request_id": 43,
|
|
"tpot_ms": 139.41879313566167,
|
|
"ttft_ms": 341.56302383496495
|
|
},
|
|
{
|
|
"e2e_ms": 17872.42465139999,
|
|
"request_id": 44,
|
|
"tpot_ms": 137.92935642744132,
|
|
"ttft_ms": 355.3963851149433
|
|
},
|
|
{
|
|
"e2e_ms": 17695.178674250878,
|
|
"request_id": 45,
|
|
"tpot_ms": 136.4247947075194,
|
|
"ttft_ms": 369.22974639591644
|
|
},
|
|
{
|
|
"e2e_ms": 17514.86814366097,
|
|
"request_id": 46,
|
|
"tpot_ms": 134.89610264554392,
|
|
"ttft_ms": 383.06310767688956
|
|
},
|
|
{
|
|
"e2e_ms": 17334.24324324006,
|
|
"request_id": 47,
|
|
"tpot_ms": 133.36661419575813,
|
|
"ttft_ms": 396.68324037877767
|
|
},
|
|
{
|
|
"e2e_ms": 17152.122611965817,
|
|
"request_id": 48,
|
|
"tpot_ms": 131.8241488699319,
|
|
"ttft_ms": 410.4557054844662
|
|
},
|
|
{
|
|
"e2e_ms": 16860.906258226023,
|
|
"request_id": 49,
|
|
"tpot_ms": 130.28066064407446,
|
|
"ttft_ms": 315.26235642856903
|
|
},
|
|
{
|
|
"e2e_ms": 16678.317507592936,
|
|
"request_id": 50,
|
|
"tpot_ms": 128.73402984160936,
|
|
"ttft_ms": 329.0957177085474
|
|
},
|
|
{
|
|
"e2e_ms": 16495.414294222966,
|
|
"request_id": 51,
|
|
"tpot_ms": 127.18492295459406,
|
|
"ttft_ms": 342.9290789895205
|
|
},
|
|
{
|
|
"e2e_ms": 16325.716031493954,
|
|
"request_id": 52,
|
|
"tpot_ms": 125.73979205687765,
|
|
"ttft_ms": 356.76244027049364
|
|
},
|
|
{
|
|
"e2e_ms": 16142.044362006885,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.18463433430247,
|
|
"ttft_ms": 370.595801550472
|
|
},
|
|
{
|
|
"e2e_ms": 15942.642543961227,
|
|
"request_id": 54,
|
|
"tpot_ms": 122.50561717425025,
|
|
"ttft_ms": 384.4291628314451
|
|
},
|
|
{
|
|
"e2e_ms": 15727.503328455541,
|
|
"request_id": 55,
|
|
"tpot_ms": 120.70436246394655,
|
|
"ttft_ms": 398.049295534328
|
|
},
|
|
{
|
|
"e2e_ms": 15403.205770995846,
|
|
"request_id": 56,
|
|
"tpot_ms": 118.90039231904261,
|
|
"ttft_ms": 302.8559464774361
|
|
},
|
|
{
|
|
"e2e_ms": 15178.030676088234,
|
|
"request_id": 57,
|
|
"tpot_ms": 117.01843597110097,
|
|
"ttft_ms": 316.6893077584092
|
|
},
|
|
{
|
|
"e2e_ms": 14941.341931765628,
|
|
"request_id": 58,
|
|
"tpot_ms": 115.04582096634839,
|
|
"ttft_ms": 330.5226690393823
|
|
},
|
|
{
|
|
"e2e_ms": 14696.772511761146,
|
|
"request_id": 59,
|
|
"tpot_ms": 113.01115339717941,
|
|
"ttft_ms": 344.3560303193607
|
|
},
|
|
{
|
|
"e2e_ms": 14451.34286402164,
|
|
"request_id": 60,
|
|
"tpot_ms": 110.96971238127013,
|
|
"ttft_ms": 358.1893916003338
|
|
},
|
|
{
|
|
"e2e_ms": 14196.293999590453,
|
|
"request_id": 61,
|
|
"tpot_ms": 108.85252950164681,
|
|
"ttft_ms": 372.02275288130693
|
|
},
|
|
{
|
|
"e2e_ms": 13938.386603616791,
|
|
"request_id": 62,
|
|
"tpot_ms": 106.71283849964966,
|
|
"ttft_ms": 385.8561141612853
|
|
},
|
|
{
|
|
"e2e_ms": 13676.049735660896,
|
|
"request_id": 63,
|
|
"tpot_ms": 104.53992523504405,
|
|
"ttft_ms": 399.4792308103001
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.125,
|
|
"passed": 8
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "3e07b28ac20428d11b8d83064d68561f03804f8adf37111de43398fdd8d94138"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 164.71713185310364,
|
|
"offered_request_rate": 1.6,
|
|
"request_metrics_sha256": "e85085ea5517c1afa9f850cb838a638360d45db695cde8d3c9fef6baf6af44c4",
|
|
"request_rate_per_gpu": 0.2,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 19447.058061600903,
|
|
"request_id": 0,
|
|
"tpot_ms": 150.8298532036487,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 19750.640261300512,
|
|
"request_id": 1,
|
|
"tpot_ms": 152.83450342712686,
|
|
"ttft_ms": 340.65832605540146
|
|
},
|
|
{
|
|
"e2e_ms": 19750.02534103151,
|
|
"request_id": 2,
|
|
"tpot_ms": 153.17602578595117,
|
|
"ttft_ms": 296.67006621570977
|
|
},
|
|
{
|
|
"e2e_ms": 19855.93358266601,
|
|
"request_id": 3,
|
|
"tpot_ms": 153.5864661294274,
|
|
"ttft_ms": 350.4523842287295
|
|
},
|
|
{
|
|
"e2e_ms": 19855.232098385564,
|
|
"request_id": 4,
|
|
"tpot_ms": 153.87573615619343,
|
|
"ttft_ms": 313.01360654900145
|
|
},
|
|
{
|
|
"e2e_ms": 19961.05377600862,
|
|
"request_id": 5,
|
|
"tpot_ms": 154.2244676371592,
|
|
"ttft_ms": 374.54638608940184
|
|
},
|
|
{
|
|
"e2e_ms": 19960.265727716724,
|
|
"request_id": 6,
|
|
"tpot_ms": 154.4718650613629,
|
|
"ttft_ms": 342.33886492363473
|
|
},
|
|
{
|
|
"e2e_ms": 19959.47767942483,
|
|
"request_id": 7,
|
|
"tpot_ms": 154.6679176552935,
|
|
"ttft_ms": 316.6521372025519
|
|
},
|
|
{
|
|
"e2e_ms": 20065.255011109733,
|
|
"request_id": 8,
|
|
"tpot_ms": 154.88778812601134,
|
|
"ttft_ms": 394.50591910629294
|
|
},
|
|
{
|
|
"e2e_ms": 20064.42261687969,
|
|
"request_id": 9,
|
|
"tpot_ms": 155.03615010045385,
|
|
"ttft_ms": 374.8315541220517
|
|
},
|
|
{
|
|
"e2e_ms": 20063.590222649644,
|
|
"request_id": 10,
|
|
"tpot_ms": 155.117974090142,
|
|
"ttft_ms": 363.6075132016101
|
|
},
|
|
{
|
|
"e2e_ms": 20062.7578284196,
|
|
"request_id": 11,
|
|
"tpot_ms": 155.13454318347627,
|
|
"ttft_ms": 360.67084411811567
|
|
},
|
|
{
|
|
"e2e_ms": 20061.925434189554,
|
|
"request_id": 12,
|
|
"tpot_ms": 155.09034115377113,
|
|
"ttft_ms": 365.45210766062206
|
|
},
|
|
{
|
|
"e2e_ms": 20061.09303995951,
|
|
"request_id": 13,
|
|
"tpot_ms": 155.09665427482798,
|
|
"ttft_ms": 363.8179470563543
|
|
},
|
|
{
|
|
"e2e_ms": 20060.260645729464,
|
|
"request_id": 14,
|
|
"tpot_ms": 155.10154332898202,
|
|
"ttft_ms": 362.3646429487479
|
|
},
|
|
{
|
|
"e2e_ms": 20059.42825149942,
|
|
"request_id": 15,
|
|
"tpot_ms": 155.10604675613317,
|
|
"ttft_ms": 360.96031347050683
|
|
},
|
|
{
|
|
"e2e_ms": 20058.595857269374,
|
|
"request_id": 16,
|
|
"tpot_ms": 155.10168661355542,
|
|
"ttft_ms": 360.6816573478344
|
|
},
|
|
{
|
|
"e2e_ms": 20057.76346303933,
|
|
"request_id": 17,
|
|
"tpot_ms": 155.0900405289204,
|
|
"ttft_ms": 361.3283158664373
|
|
},
|
|
{
|
|
"e2e_ms": 20056.931068809285,
|
|
"request_id": 18,
|
|
"tpot_ms": 155.07436184636043,
|
|
"ttft_ms": 362.4871143215085
|
|
},
|
|
{
|
|
"e2e_ms": 20056.09867457924,
|
|
"request_id": 19,
|
|
"tpot_ms": 155.0434256316313,
|
|
"ttft_ms": 365.58361936206654
|
|
},
|
|
{
|
|
"e2e_ms": 20055.26628034948,
|
|
"request_id": 20,
|
|
"tpot_ms": 154.9995151863024,
|
|
"ttft_ms": 370.32785168907446
|
|
},
|
|
{
|
|
"e2e_ms": 20054.433886119732,
|
|
"request_id": 21,
|
|
"tpot_ms": 154.96275487551375,
|
|
"ttft_ms": 374.16401692948574
|
|
},
|
|
{
|
|
"e2e_ms": 20053.601491889985,
|
|
"request_id": 22,
|
|
"tpot_ms": 154.92303029896564,
|
|
"ttft_ms": 378.3766439213494
|
|
},
|
|
{
|
|
"e2e_ms": 20052.76909766024,
|
|
"request_id": 23,
|
|
"tpot_ms": 154.88711594543284,
|
|
"ttft_ms": 382.105372590269
|
|
},
|
|
{
|
|
"e2e_ms": 20051.936703430492,
|
|
"request_id": 24,
|
|
"tpot_ms": 154.8482609975464,
|
|
"ttft_ms": 386.20755674210193
|
|
},
|
|
{
|
|
"e2e_ms": 20051.104309200746,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.85607608193587,
|
|
"ttft_ms": 384.3826467948901
|
|
},
|
|
{
|
|
"e2e_ms": 20050.271914971,
|
|
"request_id": 26,
|
|
"tpot_ms": 154.86595665257255,
|
|
"ttft_ms": 382.2954200942838
|
|
},
|
|
{
|
|
"e2e_ms": 20049.439520741253,
|
|
"request_id": 27,
|
|
"tpot_ms": 154.87441181850545,
|
|
"ttft_ms": 380.3892197910592
|
|
},
|
|
{
|
|
"e2e_ms": 20048.607126511506,
|
|
"request_id": 28,
|
|
"tpot_ms": 154.87585298283847,
|
|
"ttft_ms": 379.37379769102006
|
|
},
|
|
{
|
|
"e2e_ms": 20047.77473228176,
|
|
"request_id": 29,
|
|
"tpot_ms": 154.8790680483812,
|
|
"ttft_ms": 378.13309013734295
|
|
},
|
|
{
|
|
"e2e_ms": 20046.942338052013,
|
|
"request_id": 30,
|
|
"tpot_ms": 154.879633497606,
|
|
"ttft_ms": 377.228883856052
|
|
},
|
|
{
|
|
"e2e_ms": 20046.109943822266,
|
|
"request_id": 31,
|
|
"tpot_ms": 154.8807461594705,
|
|
"ttft_ms": 376.2551815695119
|
|
},
|
|
{
|
|
"e2e_ms": 20045.27754959252,
|
|
"request_id": 32,
|
|
"tpot_ms": 154.8790337660788,
|
|
"ttft_ms": 375.64026130051076
|
|
},
|
|
{
|
|
"e2e_ms": 19846.84310747305,
|
|
"request_id": 33,
|
|
"tpot_ms": 153.32139973576017,
|
|
"ttft_ms": 375.02534103150964
|
|
},
|
|
{
|
|
"e2e_ms": 19648.543544766126,
|
|
"request_id": 34,
|
|
"tpot_ms": 151.76482774805996,
|
|
"ttft_ms": 374.4104207625085
|
|
},
|
|
{
|
|
"e2e_ms": 19450.677394587543,
|
|
"request_id": 35,
|
|
"tpot_ms": 150.2123500638227,
|
|
"ttft_ms": 373.7089364820605
|
|
},
|
|
{
|
|
"e2e_ms": 19253.237256186894,
|
|
"request_id": 36,
|
|
"tpot_ms": 148.66322680303372,
|
|
"ttft_ms": 373.00745220161247
|
|
},
|
|
{
|
|
"e2e_ms": 19054.80482620875,
|
|
"request_id": 37,
|
|
"tpot_ms": 147.10697182912625,
|
|
"ttft_ms": 372.2194039097175
|
|
},
|
|
{
|
|
"e2e_ms": 18856.422732154002,
|
|
"request_id": 38,
|
|
"tpot_ms": 145.55111320107227,
|
|
"ttft_ms": 371.4313556178226
|
|
},
|
|
{
|
|
"e2e_ms": 18658.616312962055,
|
|
"request_id": 39,
|
|
"tpot_ms": 143.99978744595376,
|
|
"ttft_ms": 370.64330732592765
|
|
},
|
|
{
|
|
"e2e_ms": 18469.244245876795,
|
|
"request_id": 40,
|
|
"tpot_ms": 142.51522309276308,
|
|
"ttft_ms": 369.81091309588265
|
|
},
|
|
{
|
|
"e2e_ms": 18279.907702313452,
|
|
"request_id": 41,
|
|
"tpot_ms": 141.0309384523434,
|
|
"ttft_ms": 368.97851886583766
|
|
},
|
|
{
|
|
"e2e_ms": 18091.61193748763,
|
|
"request_id": 42,
|
|
"tpot_ms": 139.55484892009318,
|
|
"ttft_ms": 368.14612463579266
|
|
},
|
|
{
|
|
"e2e_ms": 17903.060652176137,
|
|
"request_id": 43,
|
|
"tpot_ms": 138.07674741551486,
|
|
"ttft_ms": 367.31373040574766
|
|
},
|
|
{
|
|
"e2e_ms": 17716.73280728946,
|
|
"request_id": 44,
|
|
"tpot_ms": 136.61615331585634,
|
|
"ttft_ms": 366.48133617570267
|
|
},
|
|
{
|
|
"e2e_ms": 17529.12437807872,
|
|
"request_id": 45,
|
|
"tpot_ms": 135.14547587506345,
|
|
"ttft_ms": 365.64894194565767
|
|
},
|
|
{
|
|
"e2e_ms": 17339.472913240785,
|
|
"request_id": 46,
|
|
"tpot_ms": 133.65871153956826,
|
|
"ttft_ms": 364.81654771561267
|
|
},
|
|
{
|
|
"e2e_ms": 17149.611868516175,
|
|
"request_id": 47,
|
|
"tpot_ms": 132.17029696874496,
|
|
"ttft_ms": 363.9841534855677
|
|
},
|
|
{
|
|
"e2e_ms": 16958.75366988868,
|
|
"request_id": 48,
|
|
"tpot_ms": 130.67403079238707,
|
|
"ttft_ms": 363.1517592555227
|
|
},
|
|
{
|
|
"e2e_ms": 16767.44725356325,
|
|
"request_id": 49,
|
|
"tpot_ms": 129.1742353428171,
|
|
"ttft_ms": 362.3193650254777
|
|
},
|
|
{
|
|
"e2e_ms": 16576.276975363853,
|
|
"request_id": 50,
|
|
"tpot_ms": 127.67551184699543,
|
|
"ttft_ms": 361.4869707954327
|
|
},
|
|
{
|
|
"e2e_ms": 16384.89705533921,
|
|
"request_id": 51,
|
|
"tpot_ms": 126.1751376281405,
|
|
"ttft_ms": 360.65457656536637
|
|
},
|
|
{
|
|
"e2e_ms": 16202.320435741867,
|
|
"request_id": 52,
|
|
"tpot_ms": 124.74408073548226,
|
|
"ttft_ms": 359.8221823356198
|
|
},
|
|
{
|
|
"e2e_ms": 16010.428211639819,
|
|
"request_id": 53,
|
|
"tpot_ms": 123.23967262625155,
|
|
"ttft_ms": 358.98978810587323
|
|
},
|
|
{
|
|
"e2e_ms": 15808.049221831383,
|
|
"request_id": 54,
|
|
"tpot_ms": 121.6526915587028,
|
|
"ttft_ms": 358.15739387612666
|
|
},
|
|
{
|
|
"e2e_ms": 15595.17863371626,
|
|
"request_id": 55,
|
|
"tpot_ms": 119.98309948086519,
|
|
"ttft_ms": 357.3249996463801
|
|
},
|
|
{
|
|
"e2e_ms": 15377.4072544157,
|
|
"request_id": 56,
|
|
"tpot_ms": 118.27491849605565,
|
|
"ttft_ms": 356.4926054166335
|
|
},
|
|
{
|
|
"e2e_ms": 15157.846080032627,
|
|
"request_id": 57,
|
|
"tpot_ms": 116.55264463658062,
|
|
"ttft_ms": 355.66021118688695
|
|
},
|
|
{
|
|
"e2e_ms": 14930.609139372891,
|
|
"request_id": 58,
|
|
"tpot_ms": 114.76993167256497,
|
|
"ttft_ms": 354.8278169571404
|
|
},
|
|
{
|
|
"e2e_ms": 14698.1184149259,
|
|
"request_id": 59,
|
|
"tpot_ms": 112.9458503322717,
|
|
"ttft_ms": 353.9954227273938
|
|
},
|
|
{
|
|
"e2e_ms": 14465.054205320868,
|
|
"request_id": 60,
|
|
"tpot_ms": 111.11725336081277,
|
|
"ttft_ms": 353.16302849764725
|
|
},
|
|
{
|
|
"e2e_ms": 14225.577184589198,
|
|
"request_id": 61,
|
|
"tpot_ms": 109.23816181355352,
|
|
"ttft_ms": 352.3306342679007
|
|
},
|
|
{
|
|
"e2e_ms": 13984.194476163324,
|
|
"request_id": 62,
|
|
"tpot_ms": 107.34406485137929,
|
|
"ttft_ms": 351.4982400381541
|
|
},
|
|
{
|
|
"e2e_ms": 13739.858786414643,
|
|
"request_id": 63,
|
|
"tpot_ms": 105.42671606776564,
|
|
"ttft_ms": 350.66584580840754
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.140625,
|
|
"passed": 9
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.4375,
|
|
"passed": 28
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "850087527f8ec5b333ae4bb00c5837c85f7dafca77de3d8c80820e071da8dfbb"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 391.55592226982117,
|
|
"offered_request_rate": 2.4,
|
|
"request_metrics_sha256": "c4ae9e9bb2050189ca506883c0c12efd5faecce0ec6249ad641d32f3b9a7f130",
|
|
"request_rate_per_gpu": 0.3,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 25856.75271176486,
|
|
"request_id": 0,
|
|
"tpot_ms": 201.29988981911293,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 26179.284600266838,
|
|
"request_id": 1,
|
|
"tpot_ms": 203.30897363251975,
|
|
"ttft_ms": 359.0449489368311
|
|
},
|
|
{
|
|
"e2e_ms": 25981.87281483019,
|
|
"request_id": 2,
|
|
"tpot_ms": 201.96786246415923,
|
|
"ttft_ms": 331.9542818819672
|
|
},
|
|
{
|
|
"e2e_ms": 25784.35394549511,
|
|
"request_id": 3,
|
|
"tpot_ms": 200.62102535110301,
|
|
"ttft_ms": 305.48372590502584
|
|
},
|
|
{
|
|
"e2e_ms": 25696.083985341404,
|
|
"request_id": 4,
|
|
"tpot_ms": 199.35042104178692,
|
|
"ttft_ms": 378.5805130344673
|
|
},
|
|
{
|
|
"e2e_ms": 25498.25455731707,
|
|
"request_id": 5,
|
|
"tpot_ms": 197.981349219281,
|
|
"ttft_ms": 354.62320646838384
|
|
},
|
|
{
|
|
"e2e_ms": 25300.295013812065,
|
|
"request_id": 6,
|
|
"tpot_ms": 196.59886557857163,
|
|
"ttft_ms": 332.2390853334691
|
|
},
|
|
{
|
|
"e2e_ms": 25102.100957670827,
|
|
"request_id": 7,
|
|
"tpot_ms": 195.19408059762435,
|
|
"ttft_ms": 312.45272177253236
|
|
},
|
|
{
|
|
"e2e_ms": 25012.968951433395,
|
|
"request_id": 8,
|
|
"tpot_ms": 193.83981874911316,
|
|
"ttft_ms": 395.3119702960262
|
|
},
|
|
{
|
|
"e2e_ms": 24814.29543067174,
|
|
"request_id": 9,
|
|
"tpot_ms": 192.41025790813484,
|
|
"ttft_ms": 378.1926763386156
|
|
},
|
|
{
|
|
"e2e_ms": 24615.47335754875,
|
|
"request_id": 10,
|
|
"tpot_ms": 190.95429488392014,
|
|
"ttft_ms": 364.2779072908935
|
|
},
|
|
{
|
|
"e2e_ms": 24416.46039907787,
|
|
"request_id": 11,
|
|
"tpot_ms": 189.47286981272285,
|
|
"ttft_ms": 353.405932862068
|
|
},
|
|
{
|
|
"e2e_ms": 24216.993764693514,
|
|
"request_id": 12,
|
|
"tpot_ms": 187.96377697549212,
|
|
"ttft_ms": 345.59408880601427
|
|
},
|
|
{
|
|
"e2e_ms": 24017.241128749425,
|
|
"request_id": 13,
|
|
"tpot_ms": 186.4682030159872,
|
|
"ttft_ms": 335.7793457190477
|
|
},
|
|
{
|
|
"e2e_ms": 23816.999491057053,
|
|
"request_id": 14,
|
|
"tpot_ms": 184.96817994362138,
|
|
"ttft_ms": 326.0406382171395
|
|
},
|
|
{
|
|
"e2e_ms": 23616.430479744246,
|
|
"request_id": 15,
|
|
"tpot_ms": 183.46465752234423,
|
|
"ttft_ms": 316.41897440652934
|
|
},
|
|
{
|
|
"e2e_ms": 23412.66959473249,
|
|
"request_id": 16,
|
|
"tpot_ms": 181.92890330417333,
|
|
"ttft_ms": 307.69887510247963
|
|
},
|
|
{
|
|
"e2e_ms": 23315.084360363784,
|
|
"request_id": 17,
|
|
"tpot_ms": 180.37041078851854,
|
|
"ttft_ms": 408.042190221928
|
|
},
|
|
{
|
|
"e2e_ms": 23111.095991700124,
|
|
"request_id": 18,
|
|
"tpot_ms": 178.82629774094855,
|
|
"ttft_ms": 400.1561785996568
|
|
},
|
|
{
|
|
"e2e_ms": 22907.001651879975,
|
|
"request_id": 19,
|
|
"tpot_ms": 177.27413055190453,
|
|
"ttft_ms": 393.18707178810007
|
|
},
|
|
{
|
|
"e2e_ms": 22702.9918523004,
|
|
"request_id": 20,
|
|
"tpot_ms": 175.71469647400258,
|
|
"ttft_ms": 387.22540010207496
|
|
},
|
|
{
|
|
"e2e_ms": 22499.024594644914,
|
|
"request_id": 21,
|
|
"tpot_ms": 174.1539791842547,
|
|
"ttft_ms": 381.46923824456815
|
|
},
|
|
{
|
|
"e2e_ms": 22294.98921706886,
|
|
"request_id": 22,
|
|
"tpot_ms": 172.59076723758398,
|
|
"ttft_ms": 375.96177789569475
|
|
},
|
|
{
|
|
"e2e_ms": 22091.055170736552,
|
|
"request_id": 23,
|
|
"tpot_ms": 171.02806584364455,
|
|
"ttft_ms": 370.49080859369354
|
|
},
|
|
{
|
|
"e2e_ms": 21886.829371806663,
|
|
"request_id": 24,
|
|
"tpot_ms": 169.4599867344259,
|
|
"ttft_ms": 365.4110565345743
|
|
},
|
|
{
|
|
"e2e_ms": 21682.517380669786,
|
|
"request_id": 25,
|
|
"tpot_ms": 167.90637996068475,
|
|
"ttft_ms": 358.40712566282207
|
|
},
|
|
{
|
|
"e2e_ms": 21478.40806232245,
|
|
"request_id": 26,
|
|
"tpot_ms": 166.35416808544093,
|
|
"ttft_ms": 351.4287154714495
|
|
},
|
|
{
|
|
"e2e_ms": 21274.47246393178,
|
|
"request_id": 27,
|
|
"tpot_ms": 164.80170050625136,
|
|
"ttft_ms": 344.6564996378552
|
|
},
|
|
{
|
|
"e2e_ms": 21070.655727841604,
|
|
"request_id": 28,
|
|
"tpot_ms": 163.24706150884583,
|
|
"ttft_ms": 338.278916218183
|
|
},
|
|
{
|
|
"e2e_ms": 20866.83848786135,
|
|
"request_id": 29,
|
|
"tpot_ms": 161.69251523328387,
|
|
"ttft_ms": 331.88905323430083
|
|
},
|
|
{
|
|
"e2e_ms": 20663.178771213603,
|
|
"request_id": 30,
|
|
"tpot_ms": 160.1348533340772,
|
|
"ttft_ms": 326.0523977857979
|
|
},
|
|
{
|
|
"e2e_ms": 20459.48769489552,
|
|
"request_id": 31,
|
|
"tpot_ms": 158.57430185728822,
|
|
"ttft_ms": 320.55135901991605
|
|
},
|
|
{
|
|
"e2e_ms": 20255.951788183756,
|
|
"request_id": 32,
|
|
"tpot_ms": 157.01541837295076,
|
|
"ttft_ms": 314.9936548190073
|
|
},
|
|
{
|
|
"e2e_ms": 20052.589009493837,
|
|
"request_id": 33,
|
|
"tpot_ms": 155.45687267778135,
|
|
"ttft_ms": 309.5661794156026
|
|
},
|
|
{
|
|
"e2e_ms": 19955.947670796646,
|
|
"request_id": 34,
|
|
"tpot_ms": 153.8983040696114,
|
|
"ttft_ms": 410.8630539559979
|
|
},
|
|
{
|
|
"e2e_ms": 19752.847929041352,
|
|
"request_id": 35,
|
|
"tpot_ms": 152.3343481355919,
|
|
"ttft_ms": 406.3857158211803
|
|
},
|
|
{
|
|
"e2e_ms": 19549.96119317403,
|
|
"request_id": 36,
|
|
"tpot_ms": 150.77184485827368,
|
|
"ttft_ms": 401.93689617327345
|
|
},
|
|
{
|
|
"e2e_ms": 19346.57831151796,
|
|
"request_id": 37,
|
|
"tpot_ms": 149.20507336304368,
|
|
"ttft_ms": 397.53399441140937
|
|
},
|
|
{
|
|
"e2e_ms": 19143.22059782458,
|
|
"request_id": 38,
|
|
"tpot_ms": 147.63442381272193,
|
|
"ttft_ms": 393.6487736088932
|
|
},
|
|
{
|
|
"e2e_ms": 18940.150721561607,
|
|
"request_id": 39,
|
|
"tpot_ms": 146.06310547094216,
|
|
"ttft_ms": 390.13632675195356
|
|
},
|
|
{
|
|
"e2e_ms": 18741.29802135198,
|
|
"request_id": 40,
|
|
"tpot_ms": 144.5231784918296,
|
|
"ttft_ms": 386.85435288961753
|
|
},
|
|
{
|
|
"e2e_ms": 18542.463082904305,
|
|
"request_id": 41,
|
|
"tpot_ms": 142.98167897793616,
|
|
"ttft_ms": 383.7898527064141
|
|
},
|
|
{
|
|
"e2e_ms": 18344.148533824395,
|
|
"request_id": 42,
|
|
"tpot_ms": 141.4401875845621,
|
|
"ttft_ms": 381.2447105850083
|
|
},
|
|
{
|
|
"e2e_ms": 18145.706224501653,
|
|
"request_id": 43,
|
|
"tpot_ms": 139.89417135289136,
|
|
"ttft_ms": 379.1464626844494
|
|
},
|
|
{
|
|
"e2e_ms": 17948.37563539231,
|
|
"request_id": 44,
|
|
"tpot_ms": 138.3533077037345,
|
|
"ttft_ms": 377.5055570180292
|
|
},
|
|
{
|
|
"e2e_ms": 17750.40475411994,
|
|
"request_id": 45,
|
|
"tpot_ms": 136.80479670989413,
|
|
"ttft_ms": 376.19557196338246
|
|
},
|
|
{
|
|
"e2e_ms": 17551.412355033975,
|
|
"request_id": 46,
|
|
"tpot_ms": 135.2471347218224,
|
|
"ttft_ms": 375.0262453625339
|
|
},
|
|
{
|
|
"e2e_ms": 17352.31516600567,
|
|
"request_id": 47,
|
|
"tpot_ms": 133.6870504253856,
|
|
"ttft_ms": 374.0597619816981
|
|
},
|
|
{
|
|
"e2e_ms": 17152.719400024922,
|
|
"request_id": 48,
|
|
"tpot_ms": 132.11931086353468,
|
|
"ttft_ms": 373.56692035601566
|
|
},
|
|
{
|
|
"e2e_ms": 16952.899525195207,
|
|
"request_id": 49,
|
|
"tpot_ms": 130.53228068835782,
|
|
"ttft_ms": 375.29987777376664
|
|
},
|
|
{
|
|
"e2e_ms": 16753.147719429508,
|
|
"request_id": 50,
|
|
"tpot_ms": 128.94354832701373,
|
|
"ttft_ms": 377.31708189876304
|
|
},
|
|
{
|
|
"e2e_ms": 16553.291092750187,
|
|
"request_id": 51,
|
|
"tpot_ms": 127.34853489391125,
|
|
"ttft_ms": 380.0271612234596
|
|
},
|
|
{
|
|
"e2e_ms": 16357.836116284518,
|
|
"request_id": 52,
|
|
"tpot_ms": 125.77762361641257,
|
|
"ttft_ms": 384.07791700011984
|
|
},
|
|
{
|
|
"e2e_ms": 16157.72333756749,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.16553823707912,
|
|
"ttft_ms": 388.6999814584407
|
|
},
|
|
{
|
|
"e2e_ms": 15952.367175996273,
|
|
"request_id": 54,
|
|
"tpot_ms": 122.51027887411597,
|
|
"ttft_ms": 393.5617589835445
|
|
},
|
|
{
|
|
"e2e_ms": 15741.765215271713,
|
|
"request_id": 55,
|
|
"tpot_ms": 120.81149800475544,
|
|
"ttft_ms": 398.70496866777216
|
|
},
|
|
{
|
|
"e2e_ms": 15528.71285895543,
|
|
"request_id": 56,
|
|
"tpot_ms": 119.08582914503211,
|
|
"ttft_ms": 404.8125575363528
|
|
},
|
|
{
|
|
"e2e_ms": 15314.765605096894,
|
|
"request_id": 57,
|
|
"tpot_ms": 117.34894161957355,
|
|
"ttft_ms": 411.4500194110526
|
|
},
|
|
{
|
|
"e2e_ms": 15096.980468100028,
|
|
"request_id": 58,
|
|
"tpot_ms": 115.58065073324867,
|
|
"ttft_ms": 418.2378249774459
|
|
},
|
|
{
|
|
"e2e_ms": 14778.441120322277,
|
|
"request_id": 59,
|
|
"tpot_ms": 113.87954551567164,
|
|
"ttft_ms": 315.7388398319796
|
|
},
|
|
{
|
|
"e2e_ms": 14557.742348852762,
|
|
"request_id": 60,
|
|
"tpot_ms": 112.08473611057838,
|
|
"ttft_ms": 322.9808628093096
|
|
},
|
|
{
|
|
"e2e_ms": 14333.83717181993,
|
|
"request_id": 61,
|
|
"tpot_ms": 110.26282336485625,
|
|
"ttft_ms": 330.4586044831872
|
|
},
|
|
{
|
|
"e2e_ms": 14108.97915094099,
|
|
"request_id": 62,
|
|
"tpot_ms": 108.43298632152924,
|
|
"ttft_ms": 337.989888106776
|
|
},
|
|
{
|
|
"e2e_ms": 13882.64463939965,
|
|
"request_id": 63,
|
|
"tpot_ms": 106.58813679106942,
|
|
"ttft_ms": 345.9512669338345
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.125,
|
|
"passed": 8
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.421875,
|
|
"passed": 27
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.71875,
|
|
"passed": 46
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "0b9f30d883de83b4af69dd114bd5d2bb974b7fd1f01d45f913bf5049e4b72a86"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 401.55159425735474,
|
|
"offered_request_rate": 3.2,
|
|
"request_metrics_sha256": "48ff1d03e0aeb45bb4395dca86e089cdbe1c7de16b8557fbe8f25e8e65c77def",
|
|
"request_rate_per_gpu": 0.4,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 26482.398681960134,
|
|
"request_id": 0,
|
|
"tpot_ms": 206.22623604112295,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 26389.27725660841,
|
|
"request_id": 1,
|
|
"tpot_ms": 204.89007083645959,
|
|
"ttft_ms": 368.23826037804076
|
|
},
|
|
{
|
|
"e2e_ms": 26186.398050214455,
|
|
"request_id": 2,
|
|
"tpot_ms": 203.43938315354237,
|
|
"ttft_ms": 349.5963897145746
|
|
},
|
|
{
|
|
"e2e_ms": 25983.46530187178,
|
|
"request_id": 3,
|
|
"tpot_ms": 201.98715121187504,
|
|
"ttft_ms": 331.0970979636474
|
|
},
|
|
{
|
|
"e2e_ms": 25780.42643937068,
|
|
"request_id": 4,
|
|
"tpot_ms": 200.53276712329495,
|
|
"ttft_ms": 312.76501471222184
|
|
},
|
|
{
|
|
"e2e_ms": 25577.34505869151,
|
|
"request_id": 5,
|
|
"tpot_ms": 199.0733214464005,
|
|
"ttft_ms": 295.0332349986446
|
|
},
|
|
{
|
|
"e2e_ms": 25483.55218185351,
|
|
"request_id": 6,
|
|
"tpot_ms": 197.69083780569105,
|
|
"ttft_ms": 376.81578053074594
|
|
},
|
|
{
|
|
"e2e_ms": 25280.28848711639,
|
|
"request_id": 7,
|
|
"tpot_ms": 196.22290463044078,
|
|
"ttft_ms": 359.97959905041466
|
|
},
|
|
{
|
|
"e2e_ms": 25076.966707259246,
|
|
"request_id": 8,
|
|
"tpot_ms": 194.75030755695653,
|
|
"ttft_ms": 343.6776475257637
|
|
},
|
|
{
|
|
"e2e_ms": 24873.46328021192,
|
|
"request_id": 9,
|
|
"tpot_ms": 193.27265662306831,
|
|
"ttft_ms": 327.83588908224147
|
|
},
|
|
{
|
|
"e2e_ms": 24669.88557698392,
|
|
"request_id": 10,
|
|
"tpot_ms": 191.78984111033463,
|
|
"ttft_ms": 312.57575597142215
|
|
},
|
|
{
|
|
"e2e_ms": 24466.212431081483,
|
|
"request_id": 11,
|
|
"tpot_ms": 190.30296300954802,
|
|
"ttft_ms": 297.7361288688827
|
|
},
|
|
{
|
|
"e2e_ms": 24370.912463364126,
|
|
"request_id": 12,
|
|
"tpot_ms": 188.79387017231738,
|
|
"ttft_ms": 394.09095147981964
|
|
},
|
|
{
|
|
"e2e_ms": 24166.86947872558,
|
|
"request_id": 13,
|
|
"tpot_ms": 187.2967209572732,
|
|
"ttft_ms": 380.1859171518851
|
|
},
|
|
{
|
|
"e2e_ms": 23962.581993212396,
|
|
"request_id": 14,
|
|
"tpot_ms": 185.79746051386414,
|
|
"ttft_ms": 366.30450795165007
|
|
},
|
|
{
|
|
"e2e_ms": 23758.13082088949,
|
|
"request_id": 15,
|
|
"tpot_ms": 184.29572160562404,
|
|
"ttft_ms": 352.57417697523687
|
|
},
|
|
{
|
|
"e2e_ms": 23552.083711717114,
|
|
"request_id": 16,
|
|
"tpot_ms": 182.7751996506789,
|
|
"ttft_ms": 339.633356080892
|
|
},
|
|
{
|
|
"e2e_ms": 23345.944189149544,
|
|
"request_id": 17,
|
|
"tpot_ms": 181.25255279330477,
|
|
"ttft_ms": 326.8699843998375
|
|
},
|
|
{
|
|
"e2e_ms": 23139.783338151214,
|
|
"request_id": 18,
|
|
"tpot_ms": 179.72694307410856,
|
|
"ttft_ms": 314.461567739424
|
|
},
|
|
{
|
|
"e2e_ms": 22933.56950157464,
|
|
"request_id": 19,
|
|
"tpot_ms": 178.1977153273412,
|
|
"ttft_ms": 302.45965500230557
|
|
},
|
|
{
|
|
"e2e_ms": 22833.726368661068,
|
|
"request_id": 20,
|
|
"tpot_ms": 176.63828124943853,
|
|
"ttft_ms": 400.6646499823745
|
|
},
|
|
{
|
|
"e2e_ms": 22627.576073166823,
|
|
"request_id": 21,
|
|
"tpot_ms": 175.09813975268048,
|
|
"ttft_ms": 390.11232457640244
|
|
},
|
|
{
|
|
"e2e_ms": 22421.391717712297,
|
|
"request_id": 22,
|
|
"tpot_ms": 173.55627478074584,
|
|
"ttft_ms": 379.7448205575735
|
|
},
|
|
{
|
|
"e2e_ms": 22215.258027879143,
|
|
"request_id": 23,
|
|
"tpot_ms": 172.01247264259936,
|
|
"ttft_ms": 369.67400226902566
|
|
},
|
|
{
|
|
"e2e_ms": 22008.978461747698,
|
|
"request_id": 24,
|
|
"tpot_ms": 170.46437149324944,
|
|
"ttft_ms": 360.0032821050201
|
|
},
|
|
{
|
|
"e2e_ms": 21802.65579951293,
|
|
"request_id": 25,
|
|
"tpot_ms": 168.91532250977505,
|
|
"ttft_ms": 350.4098407714995
|
|
},
|
|
{
|
|
"e2e_ms": 21596.434473671965,
|
|
"request_id": 26,
|
|
"tpot_ms": 167.36573728219415,
|
|
"ttft_ms": 340.98583883331025
|
|
},
|
|
{
|
|
"e2e_ms": 21390.300007810118,
|
|
"request_id": 27,
|
|
"tpot_ms": 165.81511332736412,
|
|
"ttft_ms": 331.7806152348748
|
|
},
|
|
{
|
|
"e2e_ms": 21184.224973098888,
|
|
"request_id": 28,
|
|
"tpot_ms": 164.26380332238384,
|
|
"ttft_ms": 322.72195115614187
|
|
},
|
|
{
|
|
"e2e_ms": 20978.149686441437,
|
|
"request_id": 29,
|
|
"tpot_ms": 162.71174941720716,
|
|
"ttft_ms": 313.7575104561279
|
|
},
|
|
{
|
|
"e2e_ms": 20772.153161450715,
|
|
"request_id": 30,
|
|
"tpot_ms": 161.15510654351164,
|
|
"ttft_ms": 305.4546304247392
|
|
},
|
|
{
|
|
"e2e_ms": 20672.62875180034,
|
|
"request_id": 31,
|
|
"tpot_ms": 159.59455506672822,
|
|
"ttft_ms": 404.12025832585744
|
|
},
|
|
{
|
|
"e2e_ms": 20466.69413177714,
|
|
"request_id": 32,
|
|
"tpot_ms": 158.03533915661183,
|
|
"ttft_ms": 396.2060588874419
|
|
},
|
|
{
|
|
"e2e_ms": 20260.84607576572,
|
|
"request_id": 33,
|
|
"tpot_ms": 156.475168525323,
|
|
"ttft_ms": 388.49967304969545
|
|
},
|
|
{
|
|
"e2e_ms": 20055.02118508905,
|
|
"request_id": 34,
|
|
"tpot_ms": 154.9132321630232,
|
|
"ttft_ms": 381.04070038510326
|
|
},
|
|
{
|
|
"e2e_ms": 19849.304647544788,
|
|
"request_id": 35,
|
|
"tpot_ms": 153.3461190070677,
|
|
"ttft_ms": 374.34753364718887
|
|
},
|
|
{
|
|
"e2e_ms": 19643.694612944317,
|
|
"request_id": 36,
|
|
"tpot_ms": 151.77962188577303,
|
|
"ttft_ms": 367.6826334511443
|
|
},
|
|
{
|
|
"e2e_ms": 19437.836505449406,
|
|
"request_id": 37,
|
|
"tpot_ms": 150.210341914627,
|
|
"ttft_ms": 361.1230822917744
|
|
},
|
|
{
|
|
"e2e_ms": 19231.99098193562,
|
|
"request_id": 38,
|
|
"tpot_ms": 148.63640086508497,
|
|
"ttft_ms": 355.16807206983003
|
|
},
|
|
{
|
|
"e2e_ms": 19026.289377137648,
|
|
"request_id": 39,
|
|
"tpot_ms": 147.05985988148976,
|
|
"ttft_ms": 349.6871721884496
|
|
},
|
|
{
|
|
"e2e_ms": 18822.696360365982,
|
|
"request_id": 40,
|
|
"tpot_ms": 145.49844654462956,
|
|
"ttft_ms": 344.3936491980288
|
|
},
|
|
{
|
|
"e2e_ms": 18619.112224474804,
|
|
"request_id": 41,
|
|
"tpot_ms": 143.93654065290042,
|
|
"ttft_ms": 339.17156155644966
|
|
},
|
|
{
|
|
"e2e_ms": 18415.78828326805,
|
|
"request_id": 42,
|
|
"tpot_ms": 142.37219516274803,
|
|
"ttft_ms": 334.51949759905375
|
|
},
|
|
{
|
|
"e2e_ms": 18212.400461939902,
|
|
"request_id": 43,
|
|
"tpot_ms": 140.8040960160447,
|
|
"ttft_ms": 330.280267902225
|
|
},
|
|
{
|
|
"e2e_ms": 18009.568500717836,
|
|
"request_id": 44,
|
|
"tpot_ms": 139.2366051127279,
|
|
"ttft_ms": 326.5196514013962
|
|
},
|
|
{
|
|
"e2e_ms": 17806.416393414805,
|
|
"request_id": 45,
|
|
"tpot_ms": 137.66365486442547,
|
|
"ttft_ms": 323.1322256327704
|
|
},
|
|
{
|
|
"e2e_ms": 17602.75352720573,
|
|
"request_id": 46,
|
|
"tpot_ms": 136.08599255484802,
|
|
"ttft_ms": 319.83247274003014
|
|
},
|
|
{
|
|
"e2e_ms": 17399.03826602459,
|
|
"request_id": 47,
|
|
"tpot_ms": 134.50648843613666,
|
|
"ttft_ms": 316.71423463523496
|
|
},
|
|
{
|
|
"e2e_ms": 17195.073716367715,
|
|
"request_id": 48,
|
|
"tpot_ms": 132.92201961792392,
|
|
"ttft_ms": 313.9772248913761
|
|
},
|
|
{
|
|
"e2e_ms": 16990.997112286357,
|
|
"request_id": 49,
|
|
"tpot_ms": 131.3320936883647,
|
|
"ttft_ms": 311.8212138640413
|
|
},
|
|
{
|
|
"e2e_ms": 16895.41197318666,
|
|
"request_id": 50,
|
|
"tpot_ms": 129.74336132702078,
|
|
"ttft_ms": 418.0050846550216
|
|
},
|
|
{
|
|
"e2e_ms": 16691.3169931805,
|
|
"request_id": 51,
|
|
"tpot_ms": 128.14881090811454,
|
|
"ttft_ms": 416.418007849952
|
|
},
|
|
{
|
|
"e2e_ms": 16489.422838281163,
|
|
"request_id": 52,
|
|
"tpot_ms": 126.56215930365238,
|
|
"ttft_ms": 416.0286067173118
|
|
},
|
|
{
|
|
"e2e_ms": 16285.199782255653,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.95445752714085,
|
|
"ttft_ms": 415.9836763087661
|
|
},
|
|
{
|
|
"e2e_ms": 16078.355034803544,
|
|
"request_id": 54,
|
|
"tpot_ms": 123.3249765236981,
|
|
"ttft_ms": 416.08301629388575
|
|
},
|
|
{
|
|
"e2e_ms": 15868.887387774763,
|
|
"request_id": 55,
|
|
"tpot_ms": 121.67321161824907,
|
|
"ttft_ms": 416.3895122571297
|
|
},
|
|
{
|
|
"e2e_ms": 15658.194542949623,
|
|
"request_id": 56,
|
|
"tpot_ms": 120.00563624201372,
|
|
"ttft_ms": 417.4787402138804
|
|
},
|
|
{
|
|
"e2e_ms": 15447.054249353854,
|
|
"request_id": 57,
|
|
"tpot_ms": 118.33082151865976,
|
|
"ttft_ms": 419.0399164840635
|
|
},
|
|
{
|
|
"e2e_ms": 15233.99501418892,
|
|
"request_id": 58,
|
|
"tpot_ms": 116.64063648866745,
|
|
"ttft_ms": 420.6341801281539
|
|
},
|
|
{
|
|
"e2e_ms": 15019.622333077174,
|
|
"request_id": 59,
|
|
"tpot_ms": 114.93953127109042,
|
|
"ttft_ms": 422.3018616486911
|
|
},
|
|
{
|
|
"e2e_ms": 14805.106280675915,
|
|
"request_id": 60,
|
|
"tpot_ms": 113.23514577866216,
|
|
"ttft_ms": 424.2427667858202
|
|
},
|
|
{
|
|
"e2e_ms": 14588.987025492997,
|
|
"request_id": 61,
|
|
"tpot_ms": 111.5171161341123,
|
|
"ttft_ms": 426.31327646073555
|
|
},
|
|
{
|
|
"e2e_ms": 14372.391348386529,
|
|
"request_id": 62,
|
|
"tpot_ms": 109.79194855942059,
|
|
"ttft_ms": 428.8138813401155
|
|
},
|
|
{
|
|
"e2e_ms": 14155.057425949359,
|
|
"request_id": 63,
|
|
"tpot_ms": 108.05983074662959,
|
|
"ttft_ms": 431.45892112740114
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.109375,
|
|
"passed": 7
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.40625,
|
|
"passed": 26
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.71875,
|
|
"passed": 46
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "96c4e04e8f133e66449a5567fcde568f27af44bd9f5d1cadeade66b771a8f0f3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"loads": [
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 775.6971135139465,
|
|
"offered_request_rate": 0.1,
|
|
"request_metrics_sha256": "1146954f1d73891536473385a3ecb02b14b7eaeee231291f2cbe6f9d4d222414",
|
|
"request_rate_per_gpu": 0.0125,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12579.084570749343,
|
|
"request_id": 0,
|
|
"tpot_ms": 96.75132177962064,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 12682.66705927438,
|
|
"request_id": 1,
|
|
"tpot_ms": 96.90664947750408,
|
|
"ttft_ms": 375.5225756313614
|
|
},
|
|
{
|
|
"e2e_ms": 12692.014437909165,
|
|
"request_id": 2,
|
|
"tpot_ms": 96.91832475611541,
|
|
"ttft_ms": 383.3871938825091
|
|
},
|
|
{
|
|
"e2e_ms": 12605.650216029226,
|
|
"request_id": 3,
|
|
"tpot_ms": 96.9198422551915,
|
|
"ttft_ms": 296.8302496199051
|
|
},
|
|
{
|
|
"e2e_ms": 12614.259349375281,
|
|
"request_id": 4,
|
|
"tpot_ms": 96.9125118011697,
|
|
"ttft_ms": 306.3703506267288
|
|
},
|
|
{
|
|
"e2e_ms": 12622.868482721336,
|
|
"request_id": 5,
|
|
"tpot_ms": 96.9125118011697,
|
|
"ttft_ms": 314.97948397278464
|
|
},
|
|
{
|
|
"e2e_ms": 12631.47761606615,
|
|
"request_id": 6,
|
|
"tpot_ms": 96.91251180115991,
|
|
"ttft_ms": 323.5886173188405
|
|
},
|
|
{
|
|
"e2e_ms": 12640.086749411268,
|
|
"request_id": 7,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 332.19775066687873
|
|
},
|
|
{
|
|
"e2e_ms": 12648.695882756385,
|
|
"request_id": 8,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 340.80688401199666
|
|
},
|
|
{
|
|
"e2e_ms": 12657.305016101503,
|
|
"request_id": 9,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 349.4160173571146
|
|
},
|
|
{
|
|
"e2e_ms": 12665.91414944662,
|
|
"request_id": 10,
|
|
"tpot_ms": 96.9125118011369,
|
|
"ttft_ms": 358.0251507022325
|
|
},
|
|
{
|
|
"e2e_ms": 12674.523282791739,
|
|
"request_id": 11,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 366.63428404735043
|
|
},
|
|
{
|
|
"e2e_ms": 12683.132416062932,
|
|
"request_id": 12,
|
|
"tpot_ms": 96.91251180055482,
|
|
"ttft_ms": 375.24341739246836
|
|
},
|
|
{
|
|
"e2e_ms": 12692.479794575775,
|
|
"request_id": 13,
|
|
"tpot_ms": 96.91832475507798,
|
|
"ttft_ms": 383.85255068087076
|
|
},
|
|
{
|
|
"e2e_ms": 12606.11557253071,
|
|
"request_id": 14,
|
|
"tpot_ms": 96.91984225378269,
|
|
"ttft_ms": 297.2956063003096
|
|
},
|
|
{
|
|
"e2e_ms": 12614.724705710416,
|
|
"request_id": 15,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 306.8357071486503
|
|
},
|
|
{
|
|
"e2e_ms": 12623.33383889012,
|
|
"request_id": 16,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 315.44484032835385
|
|
},
|
|
{
|
|
"e2e_ms": 12631.942972069823,
|
|
"request_id": 17,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 324.0539735080574
|
|
},
|
|
{
|
|
"e2e_ms": 12640.552105249526,
|
|
"request_id": 18,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 332.663106687761
|
|
},
|
|
{
|
|
"e2e_ms": 12649.16123842923,
|
|
"request_id": 19,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 341.2722398674646
|
|
},
|
|
{
|
|
"e2e_ms": 12657.770371608933,
|
|
"request_id": 20,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 349.88137304716815
|
|
},
|
|
{
|
|
"e2e_ms": 12666.379504788636,
|
|
"request_id": 21,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 358.49050622687173
|
|
},
|
|
{
|
|
"e2e_ms": 12674.988637968341,
|
|
"request_id": 22,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 367.0996394065753
|
|
},
|
|
{
|
|
"e2e_ms": 12683.597771148045,
|
|
"request_id": 23,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 375.7087725862789
|
|
},
|
|
{
|
|
"e2e_ms": 12692.945149660885,
|
|
"request_id": 24,
|
|
"tpot_ms": 96.91832475507798,
|
|
"ttft_ms": 384.31790576598246
|
|
},
|
|
{
|
|
"e2e_ms": 12606.580927732637,
|
|
"request_id": 25,
|
|
"tpot_ms": 96.91984225470249,
|
|
"ttft_ms": 297.7609613854213
|
|
},
|
|
{
|
|
"e2e_ms": 12615.190061213469,
|
|
"request_id": 26,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 307.3010623465393
|
|
},
|
|
{
|
|
"e2e_ms": 12623.7991946943,
|
|
"request_id": 27,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 315.9101958273709
|
|
},
|
|
{
|
|
"e2e_ms": 12632.408328175132,
|
|
"request_id": 28,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 324.5193293082025
|
|
},
|
|
{
|
|
"e2e_ms": 12641.017461655963,
|
|
"request_id": 29,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 333.1284627890341
|
|
},
|
|
{
|
|
"e2e_ms": 12649.626595136795,
|
|
"request_id": 30,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 341.73759626986566
|
|
},
|
|
{
|
|
"e2e_ms": 12658.235728617627,
|
|
"request_id": 31,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 350.34672975069725
|
|
},
|
|
{
|
|
"e2e_ms": 12666.844862098458,
|
|
"request_id": 32,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 358.95586323152884
|
|
},
|
|
{
|
|
"e2e_ms": 12675.45399557929,
|
|
"request_id": 33,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 367.5649967123604
|
|
},
|
|
{
|
|
"e2e_ms": 12684.063129060121,
|
|
"request_id": 34,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 376.174130193192
|
|
},
|
|
{
|
|
"e2e_ms": 12693.41050787034,
|
|
"request_id": 35,
|
|
"tpot_ms": 96.9183247574513,
|
|
"ttft_ms": 384.7832636740236
|
|
},
|
|
{
|
|
"e2e_ms": 12607.04628612126,
|
|
"request_id": 36,
|
|
"tpot_ms": 96.91984225613498,
|
|
"ttft_ms": 298.22631959211776
|
|
},
|
|
{
|
|
"e2e_ms": 12615.655419602092,
|
|
"request_id": 37,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 307.76642073516314
|
|
},
|
|
{
|
|
"e2e_ms": 12624.264553082923,
|
|
"request_id": 38,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 316.3755542159947
|
|
},
|
|
{
|
|
"e2e_ms": 12632.873686563755,
|
|
"request_id": 39,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 324.9846876968263
|
|
},
|
|
{
|
|
"e2e_ms": 12641.482820044586,
|
|
"request_id": 40,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 333.5938211776579
|
|
},
|
|
{
|
|
"e2e_ms": 12650.091953525418,
|
|
"request_id": 41,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 342.2029546584895
|
|
},
|
|
{
|
|
"e2e_ms": 12658.70108700625,
|
|
"request_id": 42,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 350.8120881393211
|
|
},
|
|
{
|
|
"e2e_ms": 12667.310220487081,
|
|
"request_id": 43,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 359.42122162015266
|
|
},
|
|
{
|
|
"e2e_ms": 12675.919353967913,
|
|
"request_id": 44,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 368.03035510098425
|
|
},
|
|
{
|
|
"e2e_ms": 12684.528487448744,
|
|
"request_id": 45,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 376.63948858181584
|
|
},
|
|
{
|
|
"e2e_ms": 12693.875866258964,
|
|
"request_id": 46,
|
|
"tpot_ms": 96.91832475745132,
|
|
"ttft_ms": 385.2486220626474
|
|
},
|
|
{
|
|
"e2e_ms": 12607.511644509885,
|
|
"request_id": 47,
|
|
"tpot_ms": 96.91984225613498,
|
|
"ttft_ms": 298.6916779807416
|
|
},
|
|
{
|
|
"e2e_ms": 12616.120777990716,
|
|
"request_id": 48,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 308.23177912378696
|
|
},
|
|
{
|
|
"e2e_ms": 12624.729911471548,
|
|
"request_id": 49,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 316.84091260461855
|
|
},
|
|
{
|
|
"e2e_ms": 12633.339044941862,
|
|
"request_id": 50,
|
|
"tpot_ms": 96.91251180201898,
|
|
"ttft_ms": 325.45004608545014
|
|
},
|
|
{
|
|
"e2e_ms": 12641.94817818975,
|
|
"request_id": 51,
|
|
"tpot_ms": 96.91251180018479,
|
|
"ttft_ms": 334.0591795662817
|
|
},
|
|
{
|
|
"e2e_ms": 12650.557311437638,
|
|
"request_id": 52,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 342.66831284185173
|
|
},
|
|
{
|
|
"e2e_ms": 12659.166444685525,
|
|
"request_id": 53,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 351.277446089739
|
|
},
|
|
{
|
|
"e2e_ms": 12667.775577933411,
|
|
"request_id": 54,
|
|
"tpot_ms": 96.91251179996681,
|
|
"ttft_ms": 359.88657933762624
|
|
},
|
|
{
|
|
"e2e_ms": 12676.3847111813,
|
|
"request_id": 55,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 368.4957125855135
|
|
},
|
|
{
|
|
"e2e_ms": 12684.993844429187,
|
|
"request_id": 56,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 377.10484583340076
|
|
},
|
|
{
|
|
"e2e_ms": 12694.3412230047,
|
|
"request_id": 57,
|
|
"tpot_ms": 96.91832475530246,
|
|
"ttft_ms": 385.713979081288
|
|
},
|
|
{
|
|
"e2e_ms": 12607.977001027848,
|
|
"request_id": 58,
|
|
"tpot_ms": 96.9198422540282,
|
|
"ttft_ms": 299.1570347662673
|
|
},
|
|
{
|
|
"e2e_ms": 12616.586134275734,
|
|
"request_id": 59,
|
|
"tpot_ms": 96.91251179996681,
|
|
"ttft_ms": 308.6971356799495
|
|
},
|
|
{
|
|
"e2e_ms": 12625.195267523623,
|
|
"request_id": 60,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 317.30626892783675
|
|
},
|
|
{
|
|
"e2e_ms": 12633.80440077151,
|
|
"request_id": 61,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 325.915402175724
|
|
},
|
|
{
|
|
"e2e_ms": 12642.413534019397,
|
|
"request_id": 62,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 334.52453542361127
|
|
},
|
|
{
|
|
"e2e_ms": 12436.81062947212,
|
|
"request_id": 63,
|
|
"tpot_ms": 95.22580284094978,
|
|
"ttft_ms": 343.1336686714985
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "37e166453683700c9df7abffa278431119205a29c83e92211ffa69d90e1dca0e"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 404.65002274513245,
|
|
"offered_request_rate": 0.2,
|
|
"request_metrics_sha256": "cf8a095cdf380d42316d1ad169ce9b2b45ae7d64348cd38271a332663de349ad",
|
|
"request_rate_per_gpu": 0.025,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12823.625057543633,
|
|
"request_id": 0,
|
|
"tpot_ms": 98.67683742367018,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 12920.533566322107,
|
|
"request_id": 1,
|
|
"tpot_ms": 99.10315669196875,
|
|
"ttft_ms": 334.4326664420745
|
|
},
|
|
{
|
|
"e2e_ms": 12921.724204826443,
|
|
"request_id": 2,
|
|
"tpot_ms": 99.22365133274592,
|
|
"ttft_ms": 320.32048556771286
|
|
},
|
|
{
|
|
"e2e_ms": 12922.91484333078,
|
|
"request_id": 3,
|
|
"tpot_ms": 99.22887081733072,
|
|
"ttft_ms": 320.84824952977976
|
|
},
|
|
{
|
|
"e2e_ms": 12924.105481838353,
|
|
"request_id": 4,
|
|
"tpot_ms": 99.22887081736695,
|
|
"ttft_ms": 322.0388880327505
|
|
},
|
|
{
|
|
"e2e_ms": 12925.296120352506,
|
|
"request_id": 5,
|
|
"tpot_ms": 99.22887081744425,
|
|
"ttft_ms": 323.2295265370873
|
|
},
|
|
{
|
|
"e2e_ms": 12926.486758866658,
|
|
"request_id": 6,
|
|
"tpot_ms": 99.22887081752154,
|
|
"ttft_ms": 324.420165041424
|
|
},
|
|
{
|
|
"e2e_ms": 12927.677397380812,
|
|
"request_id": 7,
|
|
"tpot_ms": 99.22887081749418,
|
|
"ttft_ms": 325.6108035590515
|
|
},
|
|
{
|
|
"e2e_ms": 12928.868035894964,
|
|
"request_id": 8,
|
|
"tpot_ms": 99.22887081749417,
|
|
"ttft_ms": 326.8014420732044
|
|
},
|
|
{
|
|
"e2e_ms": 12930.058674409118,
|
|
"request_id": 9,
|
|
"tpot_ms": 99.22887081749418,
|
|
"ttft_ms": 327.9920805873573
|
|
},
|
|
{
|
|
"e2e_ms": 12931.24931292327,
|
|
"request_id": 10,
|
|
"tpot_ms": 99.22887081749417,
|
|
"ttft_ms": 329.1827191015102
|
|
},
|
|
{
|
|
"e2e_ms": 12932.439951441467,
|
|
"request_id": 11,
|
|
"tpot_ms": 99.22887081752602,
|
|
"ttft_ms": 330.3733576156631
|
|
},
|
|
{
|
|
"e2e_ms": 12933.63058995827,
|
|
"request_id": 12,
|
|
"tpot_ms": 99.22887081754688,
|
|
"ttft_ms": 331.563996129816
|
|
},
|
|
{
|
|
"e2e_ms": 12934.821228475073,
|
|
"request_id": 13,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 332.7546346417307
|
|
},
|
|
{
|
|
"e2e_ms": 12936.011866991876,
|
|
"request_id": 14,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 333.94527315853395
|
|
},
|
|
{
|
|
"e2e_ms": 12937.20250550868,
|
|
"request_id": 15,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 335.1359116753372
|
|
},
|
|
{
|
|
"e2e_ms": 12938.393144025482,
|
|
"request_id": 16,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 336.3265501921404
|
|
},
|
|
{
|
|
"e2e_ms": 12939.583782542286,
|
|
"request_id": 17,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 337.51718870894365
|
|
},
|
|
{
|
|
"e2e_ms": 12940.77442105909,
|
|
"request_id": 18,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 338.7078272257469
|
|
},
|
|
{
|
|
"e2e_ms": 12941.965059575892,
|
|
"request_id": 19,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 339.8984657425501
|
|
},
|
|
{
|
|
"e2e_ms": 12943.155698092696,
|
|
"request_id": 20,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 341.08910425935335
|
|
},
|
|
{
|
|
"e2e_ms": 12944.346336609498,
|
|
"request_id": 21,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 342.2797427761566
|
|
},
|
|
{
|
|
"e2e_ms": 12945.536975126302,
|
|
"request_id": 22,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 343.4703812929598
|
|
},
|
|
{
|
|
"e2e_ms": 12946.727613643105,
|
|
"request_id": 23,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 344.66101980976305
|
|
},
|
|
{
|
|
"e2e_ms": 12947.918252142103,
|
|
"request_id": 24,
|
|
"tpot_ms": 99.22887081744517,
|
|
"ttft_ms": 345.8516583265663
|
|
},
|
|
{
|
|
"e2e_ms": 12949.10889064036,
|
|
"request_id": 25,
|
|
"tpot_ms": 99.22887081729912,
|
|
"ttft_ms": 347.0422968433695
|
|
},
|
|
{
|
|
"e2e_ms": 12950.299529138618,
|
|
"request_id": 26,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 348.23293534631716
|
|
},
|
|
{
|
|
"e2e_ms": 12951.490167636877,
|
|
"request_id": 27,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 349.42357384457523
|
|
},
|
|
{
|
|
"e2e_ms": 12952.680806135135,
|
|
"request_id": 28,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 350.6142123428333
|
|
},
|
|
{
|
|
"e2e_ms": 12953.871444633392,
|
|
"request_id": 29,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 351.80485084109137
|
|
},
|
|
{
|
|
"e2e_ms": 12955.06208313165,
|
|
"request_id": 30,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 352.99548933934943
|
|
},
|
|
{
|
|
"e2e_ms": 12956.25272162991,
|
|
"request_id": 31,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 354.1861278376075
|
|
},
|
|
{
|
|
"e2e_ms": 12957.443360128167,
|
|
"request_id": 32,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 355.37676633586557
|
|
},
|
|
{
|
|
"e2e_ms": 12958.633998626425,
|
|
"request_id": 33,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 356.56740483412364
|
|
},
|
|
{
|
|
"e2e_ms": 12959.824637124682,
|
|
"request_id": 34,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 357.7580433323817
|
|
},
|
|
{
|
|
"e2e_ms": 12961.015275622942,
|
|
"request_id": 35,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 358.94868183063977
|
|
},
|
|
{
|
|
"e2e_ms": 12962.2059141212,
|
|
"request_id": 36,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 360.13932032889784
|
|
},
|
|
{
|
|
"e2e_ms": 12963.396552619457,
|
|
"request_id": 37,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 361.3299588271559
|
|
},
|
|
{
|
|
"e2e_ms": 12964.587191117715,
|
|
"request_id": 38,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 362.520597325414
|
|
},
|
|
{
|
|
"e2e_ms": 12965.777829615974,
|
|
"request_id": 39,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 363.71123582367204
|
|
},
|
|
{
|
|
"e2e_ms": 12966.968468114232,
|
|
"request_id": 40,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 364.9018743219301
|
|
},
|
|
{
|
|
"e2e_ms": 12968.15910661249,
|
|
"request_id": 41,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 366.0925128201882
|
|
},
|
|
{
|
|
"e2e_ms": 12969.349745110747,
|
|
"request_id": 42,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 367.28315131844624
|
|
},
|
|
{
|
|
"e2e_ms": 12970.540383609006,
|
|
"request_id": 43,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 368.4737898167043
|
|
},
|
|
{
|
|
"e2e_ms": 12971.731022107264,
|
|
"request_id": 44,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 369.6644283149624
|
|
},
|
|
{
|
|
"e2e_ms": 12972.921660605522,
|
|
"request_id": 45,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 370.85506681322045
|
|
},
|
|
{
|
|
"e2e_ms": 12974.11229910378,
|
|
"request_id": 46,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 372.0457053114785
|
|
},
|
|
{
|
|
"e2e_ms": 12975.302937602039,
|
|
"request_id": 47,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 373.2363438097366
|
|
},
|
|
{
|
|
"e2e_ms": 12976.493576100296,
|
|
"request_id": 48,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 374.42698230799465
|
|
},
|
|
{
|
|
"e2e_ms": 12977.684214615067,
|
|
"request_id": 49,
|
|
"tpot_ms": 99.22887081739223,
|
|
"ttft_ms": 375.6176208062527
|
|
},
|
|
{
|
|
"e2e_ms": 12978.874853139188,
|
|
"request_id": 50,
|
|
"tpot_ms": 99.22887081759589,
|
|
"ttft_ms": 376.8082593045108
|
|
},
|
|
{
|
|
"e2e_ms": 12980.06549166331,
|
|
"request_id": 51,
|
|
"tpot_ms": 99.22887081779955,
|
|
"ttft_ms": 377.99889780276885
|
|
},
|
|
{
|
|
"e2e_ms": 12981.256130187432,
|
|
"request_id": 52,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 379.189536321519
|
|
},
|
|
{
|
|
"e2e_ms": 12982.446768711554,
|
|
"request_id": 53,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 380.3801748456408
|
|
},
|
|
{
|
|
"e2e_ms": 12983.637407235676,
|
|
"request_id": 54,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 381.5708133697626
|
|
},
|
|
{
|
|
"e2e_ms": 12984.828045759798,
|
|
"request_id": 55,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 382.76145189388444
|
|
},
|
|
{
|
|
"e2e_ms": 12986.01868428392,
|
|
"request_id": 56,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 383.95209041800626
|
|
},
|
|
{
|
|
"e2e_ms": 12987.209322808041,
|
|
"request_id": 57,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 385.1427289421281
|
|
},
|
|
{
|
|
"e2e_ms": 12988.399961332163,
|
|
"request_id": 58,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 386.3333674662499
|
|
},
|
|
{
|
|
"e2e_ms": 12989.881679406835,
|
|
"request_id": 59,
|
|
"tpot_ms": 99.2311627828068,
|
|
"ttft_ms": 387.52400599037173
|
|
},
|
|
{
|
|
"e2e_ms": 12991.363397481507,
|
|
"request_id": 60,
|
|
"tpot_ms": 99.23345474777176,
|
|
"ttft_ms": 388.71464451449356
|
|
},
|
|
{
|
|
"e2e_ms": 12896.836165730803,
|
|
"request_id": 61,
|
|
"tpot_ms": 99.23492287861664,
|
|
"ttft_ms": 294.0009601464908
|
|
},
|
|
{
|
|
"e2e_ms": 12689.81487892944,
|
|
"request_id": 62,
|
|
"tpot_ms": 97.59316693471084,
|
|
"ttft_ms": 295.4826782211626
|
|
},
|
|
{
|
|
"e2e_ms": 12448.118757066311,
|
|
"request_id": 63,
|
|
"tpot_ms": 95.67691266262345,
|
|
"ttft_ms": 297.1508489131338
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "e6cebfc858981b2ae4632d2c75f5e8c1f83280e7306351e15c7cd611d2664d80"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 223.00242853164673,
|
|
"offered_request_rate": 0.4,
|
|
"request_metrics_sha256": "e09d047b2039786ee02cd4c59c081ca95e5197ae65723b6170e43f6c0924921a",
|
|
"request_rate_per_gpu": 0.05,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 13556.392075191316,
|
|
"request_id": 0,
|
|
"tpot_ms": 104.4466564602661,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 13715.056219396885,
|
|
"request_id": 1,
|
|
"tpot_ms": 105.11756526634255,
|
|
"ttft_ms": 365.12543057138026
|
|
},
|
|
{
|
|
"e2e_ms": 13775.965021971426,
|
|
"request_id": 2,
|
|
"tpot_ms": 105.6336255570719,
|
|
"ttft_ms": 360.4945762232949
|
|
},
|
|
{
|
|
"e2e_ms": 13838.18765138338,
|
|
"request_id": 3,
|
|
"tpot_ms": 106.07242606904647,
|
|
"ttft_ms": 366.98954061447785
|
|
},
|
|
{
|
|
"e2e_ms": 13802.282961907104,
|
|
"request_id": 4,
|
|
"tpot_ms": 106.21712281117094,
|
|
"ttft_ms": 312.70836488839393
|
|
},
|
|
{
|
|
"e2e_ms": 13864.505591319055,
|
|
"request_id": 5,
|
|
"tpot_ms": 106.32495967758555,
|
|
"ttft_ms": 361.2357122656906
|
|
},
|
|
{
|
|
"e2e_ms": 13828.78708091747,
|
|
"request_id": 6,
|
|
"tpot_ms": 106.35892091285005,
|
|
"ttft_ms": 321.20412498551156
|
|
},
|
|
{
|
|
"e2e_ms": 13892.323156276025,
|
|
"request_id": 7,
|
|
"tpot_ms": 106.3810710779423,
|
|
"ttft_ms": 381.9271293773525
|
|
},
|
|
{
|
|
"e2e_ms": 13856.60464587857,
|
|
"request_id": 8,
|
|
"tpot_ms": 106.3914161711828,
|
|
"ttft_ms": 344.89479213835494
|
|
},
|
|
{
|
|
"e2e_ms": 13820.886135481267,
|
|
"request_id": 9,
|
|
"tpot_ms": 106.39288214818258,
|
|
"ttft_ms": 308.9901026620794
|
|
},
|
|
{
|
|
"e2e_ms": 13883.108764897528,
|
|
"request_id": 10,
|
|
"tpot_ms": 106.39288214821651,
|
|
"ttft_ms": 371.21273207403146
|
|
},
|
|
{
|
|
"e2e_ms": 13847.390254500226,
|
|
"request_id": 11,
|
|
"tpot_ms": 106.39288214825024,
|
|
"ttft_ms": 335.4942216724446
|
|
},
|
|
{
|
|
"e2e_ms": 13811.485565028235,
|
|
"request_id": 12,
|
|
"tpot_ms": 106.39288214828399,
|
|
"ttft_ms": 299.589532196169
|
|
},
|
|
{
|
|
"e2e_ms": 13873.708194444496,
|
|
"request_id": 13,
|
|
"tpot_ms": 106.38107407728566,
|
|
"ttft_ms": 363.31178662921815
|
|
},
|
|
{
|
|
"e2e_ms": 13837.989684047194,
|
|
"request_id": 14,
|
|
"tpot_ms": 106.38107407728566,
|
|
"ttft_ms": 327.5932762319158
|
|
},
|
|
{
|
|
"e2e_ms": 13901.52575941027,
|
|
"request_id": 15,
|
|
"tpot_ms": 106.39141617135505,
|
|
"ttft_ms": 389.81590564817736
|
|
},
|
|
{
|
|
"e2e_ms": 13865.807249012967,
|
|
"request_id": 16,
|
|
"tpot_ms": 106.39141617135505,
|
|
"ttft_ms": 354.097395250875
|
|
},
|
|
{
|
|
"e2e_ms": 13830.088738615665,
|
|
"request_id": 17,
|
|
"tpot_ms": 106.3928821483211,
|
|
"ttft_ms": 318.19270577888403
|
|
},
|
|
{
|
|
"e2e_ms": 13893.62481397874,
|
|
"request_id": 18,
|
|
"tpot_ms": 106.40322424239051,
|
|
"ttft_ms": 380.4153351951456
|
|
},
|
|
{
|
|
"e2e_ms": 13857.906303581438,
|
|
"request_id": 19,
|
|
"tpot_ms": 106.40322424239051,
|
|
"ttft_ms": 344.69682479784325
|
|
},
|
|
{
|
|
"e2e_ms": 13822.187793184135,
|
|
"request_id": 20,
|
|
"tpot_ms": 106.40469021935657,
|
|
"ttft_ms": 308.79213532585226
|
|
},
|
|
{
|
|
"e2e_ms": 13884.410422593844,
|
|
"request_id": 21,
|
|
"tpot_ms": 106.39288214826952,
|
|
"ttft_ms": 372.51438976361584
|
|
},
|
|
{
|
|
"e2e_ms": 13848.691912190276,
|
|
"request_id": 22,
|
|
"tpot_ms": 106.39288214822018,
|
|
"ttft_ms": 336.7958793663135
|
|
},
|
|
{
|
|
"e2e_ms": 13812.78722271334,
|
|
"request_id": 23,
|
|
"tpot_ms": 106.39288214818123,
|
|
"ttft_ms": 300.8911898943225
|
|
},
|
|
{
|
|
"e2e_ms": 13875.009852123696,
|
|
"request_id": 24,
|
|
"tpot_ms": 106.38107407709929,
|
|
"ttft_ms": 364.6134443320861
|
|
},
|
|
{
|
|
"e2e_ms": 13839.291341720127,
|
|
"request_id": 25,
|
|
"tpot_ms": 106.38107407704995,
|
|
"ttft_ms": 328.89493393478375
|
|
},
|
|
{
|
|
"e2e_ms": 13902.827417078257,
|
|
"request_id": 26,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 391.11756334850156
|
|
},
|
|
{
|
|
"e2e_ms": 13867.108906674688,
|
|
"request_id": 27,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 355.39905294493224
|
|
},
|
|
{
|
|
"e2e_ms": 13831.390396271117,
|
|
"request_id": 28,
|
|
"tpot_ms": 106.39288214805607,
|
|
"ttft_ms": 319.49436346799587
|
|
},
|
|
{
|
|
"e2e_ms": 13894.926471629247,
|
|
"request_id": 29,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 381.7169928783528
|
|
},
|
|
{
|
|
"e2e_ms": 13859.207961225678,
|
|
"request_id": 30,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 345.9984824747835
|
|
},
|
|
{
|
|
"e2e_ms": 13823.48945082211,
|
|
"request_id": 31,
|
|
"tpot_ms": 106.40469021908869,
|
|
"ttft_ms": 310.0937929978471
|
|
},
|
|
{
|
|
"e2e_ms": 13885.712080232466,
|
|
"request_id": 32,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 373.8160474293437
|
|
},
|
|
{
|
|
"e2e_ms": 13849.993569828897,
|
|
"request_id": 33,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 338.0975370257744
|
|
},
|
|
{
|
|
"e2e_ms": 13814.08888035196,
|
|
"request_id": 34,
|
|
"tpot_ms": 106.39288214805607,
|
|
"ttft_ms": 302.192847548838
|
|
},
|
|
{
|
|
"e2e_ms": 13876.311509762318,
|
|
"request_id": 35,
|
|
"tpot_ms": 106.3810740770235,
|
|
"ttft_ms": 365.9151019803346
|
|
},
|
|
{
|
|
"e2e_ms": 13840.592999358747,
|
|
"request_id": 36,
|
|
"tpot_ms": 106.38107407702348,
|
|
"ttft_ms": 330.1965915767653
|
|
},
|
|
{
|
|
"e2e_ms": 13904.129074716877,
|
|
"request_id": 37,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 392.41922098712223
|
|
},
|
|
{
|
|
"e2e_ms": 13868.410564313308,
|
|
"request_id": 38,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 356.7007105835529
|
|
},
|
|
{
|
|
"e2e_ms": 13832.69205390974,
|
|
"request_id": 39,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 320.79602110661654
|
|
},
|
|
{
|
|
"e2e_ms": 13896.228129267869,
|
|
"request_id": 40,
|
|
"tpot_ms": 106.40322424213304,
|
|
"ttft_ms": 383.0186505169735
|
|
},
|
|
{
|
|
"e2e_ms": 13860.509618864298,
|
|
"request_id": 41,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 347.30014011340415
|
|
},
|
|
{
|
|
"e2e_ms": 13824.79110846073,
|
|
"request_id": 42,
|
|
"tpot_ms": 106.40469021908868,
|
|
"ttft_ms": 311.3954506364678
|
|
},
|
|
{
|
|
"e2e_ms": 13888.32718381886,
|
|
"request_id": 43,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 375.1177050679644
|
|
},
|
|
{
|
|
"e2e_ms": 13852.60867341529,
|
|
"request_id": 44,
|
|
"tpot_ms": 106.40322424213304,
|
|
"ttft_ms": 339.39919466439505
|
|
},
|
|
{
|
|
"e2e_ms": 13816.89016301172,
|
|
"request_id": 45,
|
|
"tpot_ms": 106.40469021908868,
|
|
"ttft_ms": 303.4945051874587
|
|
},
|
|
{
|
|
"e2e_ms": 13879.11279242067,
|
|
"request_id": 46,
|
|
"tpot_ms": 106.392882148045,
|
|
"ttft_ms": 367.2167596189553
|
|
},
|
|
{
|
|
"e2e_ms": 13843.394282013549,
|
|
"request_id": 47,
|
|
"tpot_ms": 106.39288214801702,
|
|
"ttft_ms": 331.49824921538595
|
|
},
|
|
{
|
|
"e2e_ms": 13807.489592530403,
|
|
"request_id": 48,
|
|
"tpot_ms": 106.39288214796812,
|
|
"ttft_ms": 295.5935597384496
|
|
},
|
|
{
|
|
"e2e_ms": 13869.712221935884,
|
|
"request_id": 49,
|
|
"tpot_ms": 106.38107407689715,
|
|
"ttft_ms": 359.3158141699462
|
|
},
|
|
{
|
|
"e2e_ms": 13833.993711528763,
|
|
"request_id": 50,
|
|
"tpot_ms": 106.38107407686918,
|
|
"ttft_ms": 323.59730376637685
|
|
},
|
|
{
|
|
"e2e_ms": 13897.5297868798,
|
|
"request_id": 51,
|
|
"tpot_ms": 106.3914161708903,
|
|
"ttft_ms": 385.8199331767338
|
|
},
|
|
{
|
|
"e2e_ms": 13861.81127647268,
|
|
"request_id": 52,
|
|
"tpot_ms": 106.39141617089378,
|
|
"ttft_ms": 350.1014227691712
|
|
},
|
|
{
|
|
"e2e_ms": 13826.092766065556,
|
|
"request_id": 53,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 314.1967332860247
|
|
},
|
|
{
|
|
"e2e_ms": 13889.628841416596,
|
|
"request_id": 54,
|
|
"tpot_ms": 106.40322424192983,
|
|
"ttft_ms": 376.41936269150733
|
|
},
|
|
{
|
|
"e2e_ms": 13853.910331009472,
|
|
"request_id": 55,
|
|
"tpot_ms": 106.4032242419298,
|
|
"ttft_ms": 340.7008522843853
|
|
},
|
|
{
|
|
"e2e_ms": 13818.191820602351,
|
|
"request_id": 56,
|
|
"tpot_ms": 106.4046902189064,
|
|
"ttft_ms": 304.7961628012388
|
|
},
|
|
{
|
|
"e2e_ms": 13880.414450007833,
|
|
"request_id": 57,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 368.5184172283016
|
|
},
|
|
{
|
|
"e2e_ms": 13844.695939600711,
|
|
"request_id": 58,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 332.79990682117955
|
|
},
|
|
{
|
|
"e2e_ms": 13600.134841430987,
|
|
"request_id": 59,
|
|
"tpot_ms": 104.74991829994453,
|
|
"ttft_ms": 296.89521733803304
|
|
},
|
|
{
|
|
"e2e_ms": 13451.749583806417,
|
|
"request_id": 60,
|
|
"tpot_ms": 103.07978040977419,
|
|
"ttft_ms": 360.61747176509584
|
|
},
|
|
{
|
|
"e2e_ms": 13168.506714577745,
|
|
"request_id": 61,
|
|
"tpot_ms": 101.13076971039189,
|
|
"ttft_ms": 324.8989613579738
|
|
},
|
|
{
|
|
"e2e_ms": 12970.21046401943,
|
|
"request_id": 62,
|
|
"tpot_ms": 99.07943994689744,
|
|
"ttft_ms": 387.1215907634564
|
|
},
|
|
{
|
|
"e2e_ms": 12659.030247905519,
|
|
"request_id": 63,
|
|
"tpot_ms": 96.91045013818255,
|
|
"ttft_ms": 351.4030803563344
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "021442528b6c4067d63827496b66b6f530adbe9f4b29732b1aa7aab2112ced9f"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 142.2287290096283,
|
|
"offered_request_rate": 0.8,
|
|
"request_metrics_sha256": "6b246d56546536e46b4e8adfb23076b3cbf053da4b0af0288346df10122d6022",
|
|
"request_rate_per_gpu": 0.1,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 15423.90313583537,
|
|
"request_id": 0,
|
|
"tpot_ms": 119.15146796140039,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 15680.782114004433,
|
|
"request_id": 1,
|
|
"tpot_ms": 120.47488426274414,
|
|
"ttft_ms": 380.47181263592745
|
|
},
|
|
{
|
|
"e2e_ms": 15824.99160915063,
|
|
"request_id": 2,
|
|
"tpot_ms": 121.60222806639713,
|
|
"ttft_ms": 381.5086447181937
|
|
},
|
|
{
|
|
"e2e_ms": 15964.925902693089,
|
|
"request_id": 3,
|
|
"tpot_ms": 122.6571852387084,
|
|
"ttft_ms": 387.4633773771219
|
|
},
|
|
{
|
|
"e2e_ms": 15994.3969757696,
|
|
"request_id": 4,
|
|
"tpot_ms": 123.48492728290344,
|
|
"ttft_ms": 311.8112108408626
|
|
},
|
|
{
|
|
"e2e_ms": 16132.273068839513,
|
|
"request_id": 5,
|
|
"tpot_ms": 124.38017511477935,
|
|
"ttft_ms": 335.9908292625358
|
|
},
|
|
{
|
|
"e2e_ms": 16268.09096143688,
|
|
"request_id": 6,
|
|
"tpot_ms": 125.15372351796954,
|
|
"ttft_ms": 373.5680746547478
|
|
},
|
|
{
|
|
"e2e_ms": 16291.239371010352,
|
|
"request_id": 7,
|
|
"tpot_ms": 125.67763083904141,
|
|
"ttft_ms": 330.18025445209173
|
|
},
|
|
{
|
|
"e2e_ms": 16612.993923996273,
|
|
"request_id": 8,
|
|
"tpot_ms": 127.71634872045071,
|
|
"ttft_ms": 393.0176364990334
|
|
},
|
|
{
|
|
"e2e_ms": 16636.356874836954,
|
|
"request_id": 9,
|
|
"tpot_ms": 128.11558123152213,
|
|
"ttft_ms": 365.67805843364454
|
|
},
|
|
{
|
|
"e2e_ms": 16659.624174048608,
|
|
"request_id": 10,
|
|
"tpot_ms": 128.3236060799542,
|
|
"ttft_ms": 362.52620189442376
|
|
},
|
|
{
|
|
"e2e_ms": 16683.03953534565,
|
|
"request_id": 11,
|
|
"tpot_ms": 128.34365422438844,
|
|
"ttft_ms": 383.3954488483169
|
|
},
|
|
{
|
|
"e2e_ms": 16598.14552827833,
|
|
"request_id": 12,
|
|
"tpot_ms": 128.20865630197483,
|
|
"ttft_ms": 315.6461779275297
|
|
},
|
|
{
|
|
"e2e_ms": 16621.508479120537,
|
|
"request_id": 13,
|
|
"tpot_ms": 128.0747814081899,
|
|
"ttft_ms": 356.0112402804201
|
|
},
|
|
{
|
|
"e2e_ms": 16644.871429963147,
|
|
"request_id": 14,
|
|
"tpot_ms": 127.97565080664988,
|
|
"ttft_ms": 391.9637775186118
|
|
},
|
|
{
|
|
"e2e_ms": 16559.977422897646,
|
|
"request_id": 15,
|
|
"tpot_ms": 127.91134905305496,
|
|
"ttft_ms": 315.2360931596654
|
|
},
|
|
{
|
|
"e2e_ms": 16583.340373740255,
|
|
"request_id": 16,
|
|
"tpot_ms": 127.86325360239432,
|
|
"ttft_ms": 344.7071662361765
|
|
},
|
|
{
|
|
"e2e_ms": 16606.703324582868,
|
|
"request_id": 17,
|
|
"tpot_ms": 127.81515815173371,
|
|
"ttft_ms": 374.17823931268754
|
|
},
|
|
{
|
|
"e2e_ms": 16630.066275425477,
|
|
"request_id": 18,
|
|
"tpot_ms": 127.7832690040065,
|
|
"ttft_ms": 401.59111191665176
|
|
},
|
|
{
|
|
"e2e_ms": 16354.971144941388,
|
|
"request_id": 19,
|
|
"tpot_ms": 126.2873116642462,
|
|
"ttft_ms": 316.48256358212024
|
|
},
|
|
{
|
|
"e2e_ms": 16568.46874002116,
|
|
"request_id": 20,
|
|
"tpot_ms": 127.78612414854776,
|
|
"ttft_ms": 339.6309731555931
|
|
},
|
|
{
|
|
"e2e_ms": 16591.83169086377,
|
|
"request_id": 21,
|
|
"tpot_ms": 127.78612414856296,
|
|
"ttft_ms": 362.9939239962745
|
|
},
|
|
{
|
|
"e2e_ms": 16615.19464170638,
|
|
"request_id": 22,
|
|
"tpot_ms": 127.78612414857815,
|
|
"ttft_ms": 386.3568748369559
|
|
},
|
|
{
|
|
"e2e_ms": 16339.951449136564,
|
|
"request_id": 23,
|
|
"tpot_ms": 126.28847750749865,
|
|
"ttft_ms": 301.3148056842354
|
|
},
|
|
{
|
|
"e2e_ms": 16553.449044216337,
|
|
"request_id": 24,
|
|
"tpot_ms": 127.78518800972483,
|
|
"ttft_ms": 324.73016698128276
|
|
},
|
|
{
|
|
"e2e_ms": 16576.811995058946,
|
|
"request_id": 25,
|
|
"tpot_ms": 127.78477532898123,
|
|
"ttft_ms": 348.1455282783301
|
|
},
|
|
{
|
|
"e2e_ms": 16600.174945901555,
|
|
"request_id": 26,
|
|
"tpot_ms": 127.78477532898441,
|
|
"ttft_ms": 371.50847912053564
|
|
},
|
|
{
|
|
"e2e_ms": 16623.53789674417,
|
|
"request_id": 27,
|
|
"tpot_ms": 127.78477532898442,
|
|
"ttft_ms": 394.87142996314617
|
|
},
|
|
{
|
|
"e2e_ms": 16348.29470417435,
|
|
"request_id": 28,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 309.97742289764574
|
|
},
|
|
{
|
|
"e2e_ms": 16561.79229925412,
|
|
"request_id": 29,
|
|
"tpot_ms": 127.78308602766822,
|
|
"ttft_ms": 333.34037374025627
|
|
},
|
|
{
|
|
"e2e_ms": 16585.155250096734,
|
|
"request_id": 30,
|
|
"tpot_ms": 127.78308602766825,
|
|
"ttft_ms": 356.7033245828668
|
|
},
|
|
{
|
|
"e2e_ms": 16608.518200939343,
|
|
"request_id": 31,
|
|
"tpot_ms": 127.78308602766823,
|
|
"ttft_ms": 380.06627542547733
|
|
},
|
|
{
|
|
"e2e_ms": 16631.881151781956,
|
|
"request_id": 32,
|
|
"tpot_ms": 127.78360948578973,
|
|
"ttft_ms": 403.36274708666053
|
|
},
|
|
{
|
|
"e2e_ms": 16356.786021297865,
|
|
"request_id": 33,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 318.4687400211601
|
|
},
|
|
{
|
|
"e2e_ms": 16570.283616377637,
|
|
"request_id": 34,
|
|
"tpot_ms": 127.78308602766825,
|
|
"ttft_ms": 341.83169086377063
|
|
},
|
|
{
|
|
"e2e_ms": 16593.646567220247,
|
|
"request_id": 35,
|
|
"tpot_ms": 127.78308602766823,
|
|
"ttft_ms": 365.19464170638116
|
|
},
|
|
{
|
|
"e2e_ms": 16617.00951806286,
|
|
"request_id": 36,
|
|
"tpot_ms": 127.78477532898444,
|
|
"ttft_ms": 388.34305128183644
|
|
},
|
|
{
|
|
"e2e_ms": 16341.766325493041,
|
|
"request_id": 37,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 303.449044216336
|
|
},
|
|
{
|
|
"e2e_ms": 16555.263920572714,
|
|
"request_id": 38,
|
|
"tpot_ms": 127.78308602766747,
|
|
"ttft_ms": 326.81199505894654
|
|
},
|
|
{
|
|
"e2e_ms": 16578.62687141207,
|
|
"request_id": 39,
|
|
"tpot_ms": 127.78308602764184,
|
|
"ttft_ms": 350.17494590155707
|
|
},
|
|
{
|
|
"e2e_ms": 16601.989822251424,
|
|
"request_id": 40,
|
|
"tpot_ms": 127.78308602761618,
|
|
"ttft_ms": 373.5378967441676
|
|
},
|
|
{
|
|
"e2e_ms": 16625.35277309078,
|
|
"request_id": 41,
|
|
"tpot_ms": 127.78477532890675,
|
|
"ttft_ms": 396.6863063196229
|
|
},
|
|
{
|
|
"e2e_ms": 16350.109580518321,
|
|
"request_id": 42,
|
|
"tpot_ms": 126.2859628446,
|
|
"ttft_ms": 311.79229925412244
|
|
},
|
|
{
|
|
"e2e_ms": 16563.607175594883,
|
|
"request_id": 43,
|
|
"tpot_ms": 127.78308602754448,
|
|
"ttft_ms": 335.15525009673297
|
|
},
|
|
{
|
|
"e2e_ms": 16586.970126434237,
|
|
"request_id": 44,
|
|
"tpot_ms": 127.78308602751885,
|
|
"ttft_ms": 358.5182009393435
|
|
},
|
|
{
|
|
"e2e_ms": 16610.333077273594,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.78308602749323,
|
|
"ttft_ms": 381.88115178195403
|
|
},
|
|
{
|
|
"e2e_ms": 16633.69602811295,
|
|
"request_id": 46,
|
|
"tpot_ms": 127.78360948558907,
|
|
"ttft_ms": 405.17762344313724
|
|
},
|
|
{
|
|
"e2e_ms": 16548.802021044266,
|
|
"request_id": 47,
|
|
"tpot_ms": 127.78360948556401,
|
|
"ttft_ms": 320.2836163776368
|
|
},
|
|
{
|
|
"e2e_ms": 16572.164971883623,
|
|
"request_id": 48,
|
|
"tpot_ms": 127.7836094855384,
|
|
"ttft_ms": 343.64656722024733
|
|
},
|
|
{
|
|
"e2e_ms": 16595.52792272298,
|
|
"request_id": 49,
|
|
"tpot_ms": 127.78360948551277,
|
|
"ttft_ms": 367.00951806285786
|
|
},
|
|
{
|
|
"e2e_ms": 16618.890873562337,
|
|
"request_id": 50,
|
|
"tpot_ms": 127.78529878680334,
|
|
"ttft_ms": 390.15792763831314
|
|
},
|
|
{
|
|
"e2e_ms": 16343.647680989874,
|
|
"request_id": 51,
|
|
"tpot_ms": 126.28648630249732,
|
|
"ttft_ms": 305.26392057271323
|
|
},
|
|
{
|
|
"e2e_ms": 16199.563507361632,
|
|
"request_id": 52,
|
|
"tpot_ms": 124.96800500747688,
|
|
"ttft_ms": 328.6268714120695
|
|
},
|
|
{
|
|
"e2e_ms": 16032.332947106512,
|
|
"request_id": 53,
|
|
"tpot_ms": 123.46726869964634,
|
|
"ttft_ms": 351.9898222514257
|
|
},
|
|
{
|
|
"e2e_ms": 15838.885472588829,
|
|
"request_id": 54,
|
|
"tpot_ms": 121.7600999960476,
|
|
"ttft_ms": 375.35277309078197
|
|
},
|
|
{
|
|
"e2e_ms": 15619.209002299955,
|
|
"request_id": 55,
|
|
"tpot_ms": 119.84809306800307,
|
|
"ttft_ms": 398.5011826635656
|
|
},
|
|
{
|
|
"e2e_ms": 15285.473398871076,
|
|
"request_id": 56,
|
|
"tpot_ms": 117.8887104194976,
|
|
"ttft_ms": 313.60717559488194
|
|
},
|
|
{
|
|
"e2e_ms": 15049.070462905931,
|
|
"request_id": 57,
|
|
"tpot_ms": 115.84330973599758,
|
|
"ttft_ms": 336.9701264342382
|
|
},
|
|
{
|
|
"e2e_ms": 14793.47811125659,
|
|
"request_id": 58,
|
|
"tpot_ms": 113.6468112912047,
|
|
"ttft_ms": 360.3330772735944
|
|
},
|
|
{
|
|
"e2e_ms": 14524.751300129523,
|
|
"request_id": 59,
|
|
"tpot_ms": 111.34689190564231,
|
|
"ttft_ms": 383.69602811295067
|
|
},
|
|
{
|
|
"e2e_ms": 14156.606828520311,
|
|
"request_id": 60,
|
|
"tpot_ms": 109.11657328721294,
|
|
"ttft_ms": 298.802021044267
|
|
},
|
|
{
|
|
"e2e_ms": 13870.414276689999,
|
|
"request_id": 61,
|
|
"tpot_ms": 106.67912838430216,
|
|
"ttft_ms": 322.16497188362325
|
|
},
|
|
{
|
|
"e2e_ms": 13579.457505623892,
|
|
"request_id": 62,
|
|
"tpot_ms": 104.20416994410166,
|
|
"ttft_ms": 345.5279227229795
|
|
},
|
|
{
|
|
"e2e_ms": 13281.118281252404,
|
|
"request_id": 63,
|
|
"tpot_ms": 101.6710819503155,
|
|
"ttft_ms": 368.89087356233574
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.15625,
|
|
"passed": 10
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "9ed2bf69d3193253248833822a357db5130e6ca29282e93e04cc92961dba468d"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 139.56721377372742,
|
|
"offered_request_rate": 1.2,
|
|
"request_metrics_sha256": "dfafc592cb8f7c26725458265ca04cb89642135218d5f80993dfbd6e83fcea9c",
|
|
"request_rate_per_gpu": 0.15,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 17349.6420929535,
|
|
"request_id": 0,
|
|
"tpot_ms": 134.3147668363463,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 17472.38469413562,
|
|
"request_id": 1,
|
|
"tpot_ms": 135.04025977135157,
|
|
"ttft_ms": 322.27170317397037
|
|
},
|
|
{
|
|
"e2e_ms": 17594.570506858923,
|
|
"request_id": 2,
|
|
"tpot_ms": 135.72808492940857,
|
|
"ttft_ms": 357.1037208240346
|
|
},
|
|
{
|
|
"e2e_ms": 17607.2367009099,
|
|
"request_id": 3,
|
|
"tpot_ms": 136.28731175562888,
|
|
"ttft_ms": 298.7481079450336
|
|
},
|
|
{
|
|
"e2e_ms": 17921.914449676293,
|
|
"request_id": 4,
|
|
"tpot_ms": 138.39011926907332,
|
|
"ttft_ms": 346.36930250398024
|
|
},
|
|
{
|
|
"e2e_ms": 17935.406822135243,
|
|
"request_id": 5,
|
|
"tpot_ms": 138.88107821741167,
|
|
"ttft_ms": 297.50988852396
|
|
},
|
|
{
|
|
"e2e_ms": 18057.925904932094,
|
|
"request_id": 6,
|
|
"tpot_ms": 139.39521236971183,
|
|
"ttft_ms": 354.73393397869125
|
|
},
|
|
{
|
|
"e2e_ms": 18071.482157513437,
|
|
"request_id": 7,
|
|
"tpot_ms": 139.75067177492934,
|
|
"ttft_ms": 323.14684209741085
|
|
},
|
|
{
|
|
"e2e_ms": 18194.065120430685,
|
|
"request_id": 8,
|
|
"tpot_ms": 140.142283173617,
|
|
"ttft_ms": 395.99515738132584
|
|
},
|
|
{
|
|
"e2e_ms": 18207.834601589766,
|
|
"request_id": 9,
|
|
"tpot_ms": 140.4257392143267,
|
|
"ttft_ms": 373.76572137027256
|
|
},
|
|
{
|
|
"e2e_ms": 18221.607066695313,
|
|
"request_id": 10,
|
|
"tpot_ms": 140.60137526077196,
|
|
"ttft_ms": 365.232408577274
|
|
},
|
|
{
|
|
"e2e_ms": 18235.379531799863,
|
|
"request_id": 11,
|
|
"tpot_ms": 140.670460527253,
|
|
"ttft_ms": 370.23104483873226
|
|
},
|
|
{
|
|
"e2e_ms": 18249.21289308034,
|
|
"request_id": 12,
|
|
"tpot_ms": 140.64257855984852,
|
|
"ttft_ms": 387.6054159795803
|
|
},
|
|
{
|
|
"e2e_ms": 18263.046254360816,
|
|
"request_id": 13,
|
|
"tpot_ms": 140.69995603545655,
|
|
"ttft_ms": 394.1518378578337
|
|
},
|
|
{
|
|
"e2e_ms": 18276.879615640297,
|
|
"request_id": 14,
|
|
"tpot_ms": 140.75508408264292,
|
|
"ttft_ms": 400.98393714464817
|
|
},
|
|
{
|
|
"e2e_ms": 18181.536918127527,
|
|
"request_id": 15,
|
|
"tpot_ms": 140.80327015717634,
|
|
"ttft_ms": 299.52160816613116
|
|
},
|
|
{
|
|
"e2e_ms": 18195.15705083027,
|
|
"request_id": 16,
|
|
"tpot_ms": 140.84624138271937,
|
|
"ttft_ms": 307.6843952249071
|
|
},
|
|
{
|
|
"e2e_ms": 18208.77718353195,
|
|
"request_id": 17,
|
|
"tpot_ms": 140.87800086343316,
|
|
"ttft_ms": 317.2710738759417
|
|
},
|
|
{
|
|
"e2e_ms": 18222.54666469151,
|
|
"request_id": 18,
|
|
"tpot_ms": 140.90607860065492,
|
|
"ttft_ms": 327.4746824083348
|
|
},
|
|
{
|
|
"e2e_ms": 18236.319129797193,
|
|
"request_id": 19,
|
|
"tpot_ms": 140.91088449209656,
|
|
"ttft_ms": 340.6367993009294
|
|
},
|
|
{
|
|
"e2e_ms": 18250.152491077173,
|
|
"request_id": 20,
|
|
"tpot_ms": 140.8981539798204,
|
|
"ttft_ms": 356.0869356399827
|
|
},
|
|
{
|
|
"e2e_ms": 18263.985852358146,
|
|
"request_id": 21,
|
|
"tpot_ms": 140.89857595079965,
|
|
"ttft_ms": 369.8667066065901
|
|
},
|
|
{
|
|
"e2e_ms": 18277.81921363912,
|
|
"request_id": 22,
|
|
"tpot_ms": 140.90338208286406,
|
|
"ttft_ms": 383.0896891153834
|
|
},
|
|
{
|
|
"e2e_ms": 18291.6525749191,
|
|
"request_id": 23,
|
|
"tpot_ms": 140.91257237601366,
|
|
"ttft_ms": 395.7558831653678
|
|
},
|
|
{
|
|
"e2e_ms": 18305.48593620007,
|
|
"request_id": 24,
|
|
"tpot_ms": 140.9217626691632,
|
|
"ttft_ms": 408.4220772163469
|
|
},
|
|
{
|
|
"e2e_ms": 18210.143238687502,
|
|
"request_id": 25,
|
|
"tpot_ms": 140.92327164841032,
|
|
"ttft_ms": 312.8877393393914
|
|
},
|
|
{
|
|
"e2e_ms": 18223.912719846063,
|
|
"request_id": 26,
|
|
"tpot_ms": 140.92545360667498,
|
|
"ttft_ms": 326.38011179834336
|
|
},
|
|
{
|
|
"e2e_ms": 18237.68518495175,
|
|
"request_id": 27,
|
|
"tpot_ms": 140.92765906057843,
|
|
"ttft_ms": 339.8724842582901
|
|
},
|
|
{
|
|
"e2e_ms": 18251.518546232724,
|
|
"request_id": 28,
|
|
"tpot_ms": 140.92984101884323,
|
|
"ttft_ms": 353.42873683963205
|
|
},
|
|
{
|
|
"e2e_ms": 18265.3519075127,
|
|
"request_id": 29,
|
|
"tpot_ms": 140.93202297710803,
|
|
"ttft_ms": 366.98498941997923
|
|
},
|
|
{
|
|
"e2e_ms": 18279.185268793677,
|
|
"request_id": 30,
|
|
"tpot_ms": 140.93370194229104,
|
|
"ttft_ms": 380.60512212271647
|
|
},
|
|
{
|
|
"e2e_ms": 18293.01863007465,
|
|
"request_id": 31,
|
|
"tpot_ms": 140.9341814397353,
|
|
"ttft_ms": 394.3775872282629
|
|
},
|
|
{
|
|
"e2e_ms": 18306.851991354626,
|
|
"request_id": 32,
|
|
"tpot_ms": 140.93466093717961,
|
|
"ttft_ms": 408.15005233281454
|
|
},
|
|
{
|
|
"e2e_ms": 18211.509293842057,
|
|
"request_id": 33,
|
|
"tpot_ms": 140.933484965084,
|
|
"ttft_ms": 312.956703276388
|
|
},
|
|
{
|
|
"e2e_ms": 18225.278775001614,
|
|
"request_id": 34,
|
|
"tpot_ms": 140.9329819720059,
|
|
"ttft_ms": 326.79006455686374
|
|
},
|
|
{
|
|
"e2e_ms": 18239.11213628159,
|
|
"request_id": 35,
|
|
"tpot_ms": 140.93298197200983,
|
|
"ttft_ms": 340.6234258363447
|
|
},
|
|
{
|
|
"e2e_ms": 18252.945497562563,
|
|
"request_id": 36,
|
|
"tpot_ms": 140.93298197201372,
|
|
"ttft_ms": 354.45678711682046
|
|
},
|
|
{
|
|
"e2e_ms": 18266.778858843536,
|
|
"request_id": 37,
|
|
"tpot_ms": 140.93466093719667,
|
|
"ttft_ms": 368.0769198195577
|
|
},
|
|
{
|
|
"e2e_ms": 18280.612220123516,
|
|
"request_id": 38,
|
|
"tpot_ms": 140.93633990237933,
|
|
"ttft_ms": 381.69705252133923
|
|
},
|
|
{
|
|
"e2e_ms": 18294.44558140449,
|
|
"request_id": 39,
|
|
"tpot_ms": 140.93801886756114,
|
|
"ttft_ms": 395.3171852242221
|
|
},
|
|
{
|
|
"e2e_ms": 18308.27894268546,
|
|
"request_id": 40,
|
|
"tpot_ms": 140.93849836500434,
|
|
"ttft_ms": 409.0896503299106
|
|
},
|
|
{
|
|
"e2e_ms": 18212.936245171895,
|
|
"request_id": 41,
|
|
"tpot_ms": 140.93732239290452,
|
|
"ttft_ms": 313.8963012730187
|
|
},
|
|
{
|
|
"e2e_ms": 18226.708710277584,
|
|
"request_id": 42,
|
|
"tpot_ms": 140.93684289546135,
|
|
"ttft_ms": 327.72966255399183
|
|
},
|
|
{
|
|
"e2e_ms": 18047.749752064,
|
|
"request_id": 43,
|
|
"tpot_ms": 139.41879313566167,
|
|
"ttft_ms": 341.56302383496495
|
|
},
|
|
{
|
|
"e2e_ms": 17872.42465139999,
|
|
"request_id": 44,
|
|
"tpot_ms": 137.92935642744132,
|
|
"ttft_ms": 355.3963851149433
|
|
},
|
|
{
|
|
"e2e_ms": 17695.178674250878,
|
|
"request_id": 45,
|
|
"tpot_ms": 136.4247947075194,
|
|
"ttft_ms": 369.22974639591644
|
|
},
|
|
{
|
|
"e2e_ms": 17514.86814366097,
|
|
"request_id": 46,
|
|
"tpot_ms": 134.89610264554392,
|
|
"ttft_ms": 383.06310767688956
|
|
},
|
|
{
|
|
"e2e_ms": 17334.24324324006,
|
|
"request_id": 47,
|
|
"tpot_ms": 133.36661419575813,
|
|
"ttft_ms": 396.68324037877767
|
|
},
|
|
{
|
|
"e2e_ms": 17152.122611965817,
|
|
"request_id": 48,
|
|
"tpot_ms": 131.8241488699319,
|
|
"ttft_ms": 410.4557054844662
|
|
},
|
|
{
|
|
"e2e_ms": 16860.906258226023,
|
|
"request_id": 49,
|
|
"tpot_ms": 130.28066064407446,
|
|
"ttft_ms": 315.26235642856903
|
|
},
|
|
{
|
|
"e2e_ms": 16678.317507592936,
|
|
"request_id": 50,
|
|
"tpot_ms": 128.73402984160936,
|
|
"ttft_ms": 329.0957177085474
|
|
},
|
|
{
|
|
"e2e_ms": 16495.414294222966,
|
|
"request_id": 51,
|
|
"tpot_ms": 127.18492295459406,
|
|
"ttft_ms": 342.9290789895205
|
|
},
|
|
{
|
|
"e2e_ms": 16325.716031493954,
|
|
"request_id": 52,
|
|
"tpot_ms": 125.73979205687765,
|
|
"ttft_ms": 356.76244027049364
|
|
},
|
|
{
|
|
"e2e_ms": 16142.044362006885,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.18463433430247,
|
|
"ttft_ms": 370.595801550472
|
|
},
|
|
{
|
|
"e2e_ms": 15942.642543961227,
|
|
"request_id": 54,
|
|
"tpot_ms": 122.50561717425025,
|
|
"ttft_ms": 384.4291628314451
|
|
},
|
|
{
|
|
"e2e_ms": 15727.503328455541,
|
|
"request_id": 55,
|
|
"tpot_ms": 120.70436246394655,
|
|
"ttft_ms": 398.049295534328
|
|
},
|
|
{
|
|
"e2e_ms": 15403.205770995846,
|
|
"request_id": 56,
|
|
"tpot_ms": 118.90039231904261,
|
|
"ttft_ms": 302.8559464774361
|
|
},
|
|
{
|
|
"e2e_ms": 15178.030676088234,
|
|
"request_id": 57,
|
|
"tpot_ms": 117.01843597110097,
|
|
"ttft_ms": 316.6893077584092
|
|
},
|
|
{
|
|
"e2e_ms": 14941.341931765628,
|
|
"request_id": 58,
|
|
"tpot_ms": 115.04582096634839,
|
|
"ttft_ms": 330.5226690393823
|
|
},
|
|
{
|
|
"e2e_ms": 14696.772511761146,
|
|
"request_id": 59,
|
|
"tpot_ms": 113.01115339717941,
|
|
"ttft_ms": 344.3560303193607
|
|
},
|
|
{
|
|
"e2e_ms": 14451.34286402164,
|
|
"request_id": 60,
|
|
"tpot_ms": 110.96971238127013,
|
|
"ttft_ms": 358.1893916003338
|
|
},
|
|
{
|
|
"e2e_ms": 14196.293999590453,
|
|
"request_id": 61,
|
|
"tpot_ms": 108.85252950164681,
|
|
"ttft_ms": 372.02275288130693
|
|
},
|
|
{
|
|
"e2e_ms": 13938.386603616791,
|
|
"request_id": 62,
|
|
"tpot_ms": 106.71283849964966,
|
|
"ttft_ms": 385.8561141612853
|
|
},
|
|
{
|
|
"e2e_ms": 13676.049735660896,
|
|
"request_id": 63,
|
|
"tpot_ms": 104.53992523504405,
|
|
"ttft_ms": 399.4792308103001
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.125,
|
|
"passed": 8
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "3e07b28ac20428d11b8d83064d68561f03804f8adf37111de43398fdd8d94138"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 167.9100992679596,
|
|
"offered_request_rate": 1.6,
|
|
"request_metrics_sha256": "e85085ea5517c1afa9f850cb838a638360d45db695cde8d3c9fef6baf6af44c4",
|
|
"request_rate_per_gpu": 0.2,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 19447.058061600903,
|
|
"request_id": 0,
|
|
"tpot_ms": 150.8298532036487,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 19750.640261300512,
|
|
"request_id": 1,
|
|
"tpot_ms": 152.83450342712686,
|
|
"ttft_ms": 340.65832605540146
|
|
},
|
|
{
|
|
"e2e_ms": 19750.02534103151,
|
|
"request_id": 2,
|
|
"tpot_ms": 153.17602578595117,
|
|
"ttft_ms": 296.67006621570977
|
|
},
|
|
{
|
|
"e2e_ms": 19855.93358266601,
|
|
"request_id": 3,
|
|
"tpot_ms": 153.5864661294274,
|
|
"ttft_ms": 350.4523842287295
|
|
},
|
|
{
|
|
"e2e_ms": 19855.232098385564,
|
|
"request_id": 4,
|
|
"tpot_ms": 153.87573615619343,
|
|
"ttft_ms": 313.01360654900145
|
|
},
|
|
{
|
|
"e2e_ms": 19961.05377600862,
|
|
"request_id": 5,
|
|
"tpot_ms": 154.2244676371592,
|
|
"ttft_ms": 374.54638608940184
|
|
},
|
|
{
|
|
"e2e_ms": 19960.265727716724,
|
|
"request_id": 6,
|
|
"tpot_ms": 154.4718650613629,
|
|
"ttft_ms": 342.33886492363473
|
|
},
|
|
{
|
|
"e2e_ms": 19959.47767942483,
|
|
"request_id": 7,
|
|
"tpot_ms": 154.6679176552935,
|
|
"ttft_ms": 316.6521372025519
|
|
},
|
|
{
|
|
"e2e_ms": 20065.255011109733,
|
|
"request_id": 8,
|
|
"tpot_ms": 154.88778812601134,
|
|
"ttft_ms": 394.50591910629294
|
|
},
|
|
{
|
|
"e2e_ms": 20064.42261687969,
|
|
"request_id": 9,
|
|
"tpot_ms": 155.03615010045385,
|
|
"ttft_ms": 374.8315541220517
|
|
},
|
|
{
|
|
"e2e_ms": 20063.590222649644,
|
|
"request_id": 10,
|
|
"tpot_ms": 155.117974090142,
|
|
"ttft_ms": 363.6075132016101
|
|
},
|
|
{
|
|
"e2e_ms": 20062.7578284196,
|
|
"request_id": 11,
|
|
"tpot_ms": 155.13454318347627,
|
|
"ttft_ms": 360.67084411811567
|
|
},
|
|
{
|
|
"e2e_ms": 20061.925434189554,
|
|
"request_id": 12,
|
|
"tpot_ms": 155.09034115377113,
|
|
"ttft_ms": 365.45210766062206
|
|
},
|
|
{
|
|
"e2e_ms": 20061.09303995951,
|
|
"request_id": 13,
|
|
"tpot_ms": 155.09665427482798,
|
|
"ttft_ms": 363.8179470563543
|
|
},
|
|
{
|
|
"e2e_ms": 20060.260645729464,
|
|
"request_id": 14,
|
|
"tpot_ms": 155.10154332898202,
|
|
"ttft_ms": 362.3646429487479
|
|
},
|
|
{
|
|
"e2e_ms": 20059.42825149942,
|
|
"request_id": 15,
|
|
"tpot_ms": 155.10604675613317,
|
|
"ttft_ms": 360.96031347050683
|
|
},
|
|
{
|
|
"e2e_ms": 20058.595857269374,
|
|
"request_id": 16,
|
|
"tpot_ms": 155.10168661355542,
|
|
"ttft_ms": 360.6816573478344
|
|
},
|
|
{
|
|
"e2e_ms": 20057.76346303933,
|
|
"request_id": 17,
|
|
"tpot_ms": 155.0900405289204,
|
|
"ttft_ms": 361.3283158664373
|
|
},
|
|
{
|
|
"e2e_ms": 20056.931068809285,
|
|
"request_id": 18,
|
|
"tpot_ms": 155.07436184636043,
|
|
"ttft_ms": 362.4871143215085
|
|
},
|
|
{
|
|
"e2e_ms": 20056.09867457924,
|
|
"request_id": 19,
|
|
"tpot_ms": 155.0434256316313,
|
|
"ttft_ms": 365.58361936206654
|
|
},
|
|
{
|
|
"e2e_ms": 20055.26628034948,
|
|
"request_id": 20,
|
|
"tpot_ms": 154.9995151863024,
|
|
"ttft_ms": 370.32785168907446
|
|
},
|
|
{
|
|
"e2e_ms": 20054.433886119732,
|
|
"request_id": 21,
|
|
"tpot_ms": 154.96275487551375,
|
|
"ttft_ms": 374.16401692948574
|
|
},
|
|
{
|
|
"e2e_ms": 20053.601491889985,
|
|
"request_id": 22,
|
|
"tpot_ms": 154.92303029896564,
|
|
"ttft_ms": 378.3766439213494
|
|
},
|
|
{
|
|
"e2e_ms": 20052.76909766024,
|
|
"request_id": 23,
|
|
"tpot_ms": 154.88711594543284,
|
|
"ttft_ms": 382.105372590269
|
|
},
|
|
{
|
|
"e2e_ms": 20051.936703430492,
|
|
"request_id": 24,
|
|
"tpot_ms": 154.8482609975464,
|
|
"ttft_ms": 386.20755674210193
|
|
},
|
|
{
|
|
"e2e_ms": 20051.104309200746,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.85607608193587,
|
|
"ttft_ms": 384.3826467948901
|
|
},
|
|
{
|
|
"e2e_ms": 20050.271914971,
|
|
"request_id": 26,
|
|
"tpot_ms": 154.86595665257255,
|
|
"ttft_ms": 382.2954200942838
|
|
},
|
|
{
|
|
"e2e_ms": 20049.439520741253,
|
|
"request_id": 27,
|
|
"tpot_ms": 154.87441181850545,
|
|
"ttft_ms": 380.3892197910592
|
|
},
|
|
{
|
|
"e2e_ms": 20048.607126511506,
|
|
"request_id": 28,
|
|
"tpot_ms": 154.87585298283847,
|
|
"ttft_ms": 379.37379769102006
|
|
},
|
|
{
|
|
"e2e_ms": 20047.77473228176,
|
|
"request_id": 29,
|
|
"tpot_ms": 154.8790680483812,
|
|
"ttft_ms": 378.13309013734295
|
|
},
|
|
{
|
|
"e2e_ms": 20046.942338052013,
|
|
"request_id": 30,
|
|
"tpot_ms": 154.879633497606,
|
|
"ttft_ms": 377.228883856052
|
|
},
|
|
{
|
|
"e2e_ms": 20046.109943822266,
|
|
"request_id": 31,
|
|
"tpot_ms": 154.8807461594705,
|
|
"ttft_ms": 376.2551815695119
|
|
},
|
|
{
|
|
"e2e_ms": 20045.27754959252,
|
|
"request_id": 32,
|
|
"tpot_ms": 154.8790337660788,
|
|
"ttft_ms": 375.64026130051076
|
|
},
|
|
{
|
|
"e2e_ms": 19846.84310747305,
|
|
"request_id": 33,
|
|
"tpot_ms": 153.32139973576017,
|
|
"ttft_ms": 375.02534103150964
|
|
},
|
|
{
|
|
"e2e_ms": 19648.543544766126,
|
|
"request_id": 34,
|
|
"tpot_ms": 151.76482774805996,
|
|
"ttft_ms": 374.4104207625085
|
|
},
|
|
{
|
|
"e2e_ms": 19450.677394587543,
|
|
"request_id": 35,
|
|
"tpot_ms": 150.2123500638227,
|
|
"ttft_ms": 373.7089364820605
|
|
},
|
|
{
|
|
"e2e_ms": 19253.237256186894,
|
|
"request_id": 36,
|
|
"tpot_ms": 148.66322680303372,
|
|
"ttft_ms": 373.00745220161247
|
|
},
|
|
{
|
|
"e2e_ms": 19054.80482620875,
|
|
"request_id": 37,
|
|
"tpot_ms": 147.10697182912625,
|
|
"ttft_ms": 372.2194039097175
|
|
},
|
|
{
|
|
"e2e_ms": 18856.422732154002,
|
|
"request_id": 38,
|
|
"tpot_ms": 145.55111320107227,
|
|
"ttft_ms": 371.4313556178226
|
|
},
|
|
{
|
|
"e2e_ms": 18658.616312962055,
|
|
"request_id": 39,
|
|
"tpot_ms": 143.99978744595376,
|
|
"ttft_ms": 370.64330732592765
|
|
},
|
|
{
|
|
"e2e_ms": 18469.244245876795,
|
|
"request_id": 40,
|
|
"tpot_ms": 142.51522309276308,
|
|
"ttft_ms": 369.81091309588265
|
|
},
|
|
{
|
|
"e2e_ms": 18279.907702313452,
|
|
"request_id": 41,
|
|
"tpot_ms": 141.0309384523434,
|
|
"ttft_ms": 368.97851886583766
|
|
},
|
|
{
|
|
"e2e_ms": 18091.61193748763,
|
|
"request_id": 42,
|
|
"tpot_ms": 139.55484892009318,
|
|
"ttft_ms": 368.14612463579266
|
|
},
|
|
{
|
|
"e2e_ms": 17903.060652176137,
|
|
"request_id": 43,
|
|
"tpot_ms": 138.07674741551486,
|
|
"ttft_ms": 367.31373040574766
|
|
},
|
|
{
|
|
"e2e_ms": 17716.73280728946,
|
|
"request_id": 44,
|
|
"tpot_ms": 136.61615331585634,
|
|
"ttft_ms": 366.48133617570267
|
|
},
|
|
{
|
|
"e2e_ms": 17529.12437807872,
|
|
"request_id": 45,
|
|
"tpot_ms": 135.14547587506345,
|
|
"ttft_ms": 365.64894194565767
|
|
},
|
|
{
|
|
"e2e_ms": 17339.472913240785,
|
|
"request_id": 46,
|
|
"tpot_ms": 133.65871153956826,
|
|
"ttft_ms": 364.81654771561267
|
|
},
|
|
{
|
|
"e2e_ms": 17149.611868516175,
|
|
"request_id": 47,
|
|
"tpot_ms": 132.17029696874496,
|
|
"ttft_ms": 363.9841534855677
|
|
},
|
|
{
|
|
"e2e_ms": 16958.75366988868,
|
|
"request_id": 48,
|
|
"tpot_ms": 130.67403079238707,
|
|
"ttft_ms": 363.1517592555227
|
|
},
|
|
{
|
|
"e2e_ms": 16767.44725356325,
|
|
"request_id": 49,
|
|
"tpot_ms": 129.1742353428171,
|
|
"ttft_ms": 362.3193650254777
|
|
},
|
|
{
|
|
"e2e_ms": 16576.276975363853,
|
|
"request_id": 50,
|
|
"tpot_ms": 127.67551184699543,
|
|
"ttft_ms": 361.4869707954327
|
|
},
|
|
{
|
|
"e2e_ms": 16384.89705533921,
|
|
"request_id": 51,
|
|
"tpot_ms": 126.1751376281405,
|
|
"ttft_ms": 360.65457656536637
|
|
},
|
|
{
|
|
"e2e_ms": 16202.320435741867,
|
|
"request_id": 52,
|
|
"tpot_ms": 124.74408073548226,
|
|
"ttft_ms": 359.8221823356198
|
|
},
|
|
{
|
|
"e2e_ms": 16010.428211639819,
|
|
"request_id": 53,
|
|
"tpot_ms": 123.23967262625155,
|
|
"ttft_ms": 358.98978810587323
|
|
},
|
|
{
|
|
"e2e_ms": 15808.049221831383,
|
|
"request_id": 54,
|
|
"tpot_ms": 121.6526915587028,
|
|
"ttft_ms": 358.15739387612666
|
|
},
|
|
{
|
|
"e2e_ms": 15595.17863371626,
|
|
"request_id": 55,
|
|
"tpot_ms": 119.98309948086519,
|
|
"ttft_ms": 357.3249996463801
|
|
},
|
|
{
|
|
"e2e_ms": 15377.4072544157,
|
|
"request_id": 56,
|
|
"tpot_ms": 118.27491849605565,
|
|
"ttft_ms": 356.4926054166335
|
|
},
|
|
{
|
|
"e2e_ms": 15157.846080032627,
|
|
"request_id": 57,
|
|
"tpot_ms": 116.55264463658062,
|
|
"ttft_ms": 355.66021118688695
|
|
},
|
|
{
|
|
"e2e_ms": 14930.609139372891,
|
|
"request_id": 58,
|
|
"tpot_ms": 114.76993167256497,
|
|
"ttft_ms": 354.8278169571404
|
|
},
|
|
{
|
|
"e2e_ms": 14698.1184149259,
|
|
"request_id": 59,
|
|
"tpot_ms": 112.9458503322717,
|
|
"ttft_ms": 353.9954227273938
|
|
},
|
|
{
|
|
"e2e_ms": 14465.054205320868,
|
|
"request_id": 60,
|
|
"tpot_ms": 111.11725336081277,
|
|
"ttft_ms": 353.16302849764725
|
|
},
|
|
{
|
|
"e2e_ms": 14225.577184589198,
|
|
"request_id": 61,
|
|
"tpot_ms": 109.23816181355352,
|
|
"ttft_ms": 352.3306342679007
|
|
},
|
|
{
|
|
"e2e_ms": 13984.194476163324,
|
|
"request_id": 62,
|
|
"tpot_ms": 107.34406485137929,
|
|
"ttft_ms": 351.4982400381541
|
|
},
|
|
{
|
|
"e2e_ms": 13739.858786414643,
|
|
"request_id": 63,
|
|
"tpot_ms": 105.42671606776564,
|
|
"ttft_ms": 350.66584580840754
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.140625,
|
|
"passed": 9
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.4375,
|
|
"passed": 28
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "850087527f8ec5b333ae4bb00c5837c85f7dafca77de3d8c80820e071da8dfbb"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 390.6987729072571,
|
|
"offered_request_rate": 2.4,
|
|
"request_metrics_sha256": "c4ae9e9bb2050189ca506883c0c12efd5faecce0ec6249ad641d32f3b9a7f130",
|
|
"request_rate_per_gpu": 0.3,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 25856.75271176486,
|
|
"request_id": 0,
|
|
"tpot_ms": 201.29988981911293,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 26179.284600266838,
|
|
"request_id": 1,
|
|
"tpot_ms": 203.30897363251975,
|
|
"ttft_ms": 359.0449489368311
|
|
},
|
|
{
|
|
"e2e_ms": 25981.87281483019,
|
|
"request_id": 2,
|
|
"tpot_ms": 201.96786246415923,
|
|
"ttft_ms": 331.9542818819672
|
|
},
|
|
{
|
|
"e2e_ms": 25784.35394549511,
|
|
"request_id": 3,
|
|
"tpot_ms": 200.62102535110301,
|
|
"ttft_ms": 305.48372590502584
|
|
},
|
|
{
|
|
"e2e_ms": 25696.083985341404,
|
|
"request_id": 4,
|
|
"tpot_ms": 199.35042104178692,
|
|
"ttft_ms": 378.5805130344673
|
|
},
|
|
{
|
|
"e2e_ms": 25498.25455731707,
|
|
"request_id": 5,
|
|
"tpot_ms": 197.981349219281,
|
|
"ttft_ms": 354.62320646838384
|
|
},
|
|
{
|
|
"e2e_ms": 25300.295013812065,
|
|
"request_id": 6,
|
|
"tpot_ms": 196.59886557857163,
|
|
"ttft_ms": 332.2390853334691
|
|
},
|
|
{
|
|
"e2e_ms": 25102.100957670827,
|
|
"request_id": 7,
|
|
"tpot_ms": 195.19408059762435,
|
|
"ttft_ms": 312.45272177253236
|
|
},
|
|
{
|
|
"e2e_ms": 25012.968951433395,
|
|
"request_id": 8,
|
|
"tpot_ms": 193.83981874911316,
|
|
"ttft_ms": 395.3119702960262
|
|
},
|
|
{
|
|
"e2e_ms": 24814.29543067174,
|
|
"request_id": 9,
|
|
"tpot_ms": 192.41025790813484,
|
|
"ttft_ms": 378.1926763386156
|
|
},
|
|
{
|
|
"e2e_ms": 24615.47335754875,
|
|
"request_id": 10,
|
|
"tpot_ms": 190.95429488392014,
|
|
"ttft_ms": 364.2779072908935
|
|
},
|
|
{
|
|
"e2e_ms": 24416.46039907787,
|
|
"request_id": 11,
|
|
"tpot_ms": 189.47286981272285,
|
|
"ttft_ms": 353.405932862068
|
|
},
|
|
{
|
|
"e2e_ms": 24216.993764693514,
|
|
"request_id": 12,
|
|
"tpot_ms": 187.96377697549212,
|
|
"ttft_ms": 345.59408880601427
|
|
},
|
|
{
|
|
"e2e_ms": 24017.241128749425,
|
|
"request_id": 13,
|
|
"tpot_ms": 186.4682030159872,
|
|
"ttft_ms": 335.7793457190477
|
|
},
|
|
{
|
|
"e2e_ms": 23816.999491057053,
|
|
"request_id": 14,
|
|
"tpot_ms": 184.96817994362138,
|
|
"ttft_ms": 326.0406382171395
|
|
},
|
|
{
|
|
"e2e_ms": 23616.430479744246,
|
|
"request_id": 15,
|
|
"tpot_ms": 183.46465752234423,
|
|
"ttft_ms": 316.41897440652934
|
|
},
|
|
{
|
|
"e2e_ms": 23412.66959473249,
|
|
"request_id": 16,
|
|
"tpot_ms": 181.92890330417333,
|
|
"ttft_ms": 307.69887510247963
|
|
},
|
|
{
|
|
"e2e_ms": 23315.084360363784,
|
|
"request_id": 17,
|
|
"tpot_ms": 180.37041078851854,
|
|
"ttft_ms": 408.042190221928
|
|
},
|
|
{
|
|
"e2e_ms": 23111.095991700124,
|
|
"request_id": 18,
|
|
"tpot_ms": 178.82629774094855,
|
|
"ttft_ms": 400.1561785996568
|
|
},
|
|
{
|
|
"e2e_ms": 22907.001651879975,
|
|
"request_id": 19,
|
|
"tpot_ms": 177.27413055190453,
|
|
"ttft_ms": 393.18707178810007
|
|
},
|
|
{
|
|
"e2e_ms": 22702.9918523004,
|
|
"request_id": 20,
|
|
"tpot_ms": 175.71469647400258,
|
|
"ttft_ms": 387.22540010207496
|
|
},
|
|
{
|
|
"e2e_ms": 22499.024594644914,
|
|
"request_id": 21,
|
|
"tpot_ms": 174.1539791842547,
|
|
"ttft_ms": 381.46923824456815
|
|
},
|
|
{
|
|
"e2e_ms": 22294.98921706886,
|
|
"request_id": 22,
|
|
"tpot_ms": 172.59076723758398,
|
|
"ttft_ms": 375.96177789569475
|
|
},
|
|
{
|
|
"e2e_ms": 22091.055170736552,
|
|
"request_id": 23,
|
|
"tpot_ms": 171.02806584364455,
|
|
"ttft_ms": 370.49080859369354
|
|
},
|
|
{
|
|
"e2e_ms": 21886.829371806663,
|
|
"request_id": 24,
|
|
"tpot_ms": 169.4599867344259,
|
|
"ttft_ms": 365.4110565345743
|
|
},
|
|
{
|
|
"e2e_ms": 21682.517380669786,
|
|
"request_id": 25,
|
|
"tpot_ms": 167.90637996068475,
|
|
"ttft_ms": 358.40712566282207
|
|
},
|
|
{
|
|
"e2e_ms": 21478.40806232245,
|
|
"request_id": 26,
|
|
"tpot_ms": 166.35416808544093,
|
|
"ttft_ms": 351.4287154714495
|
|
},
|
|
{
|
|
"e2e_ms": 21274.47246393178,
|
|
"request_id": 27,
|
|
"tpot_ms": 164.80170050625136,
|
|
"ttft_ms": 344.6564996378552
|
|
},
|
|
{
|
|
"e2e_ms": 21070.655727841604,
|
|
"request_id": 28,
|
|
"tpot_ms": 163.24706150884583,
|
|
"ttft_ms": 338.278916218183
|
|
},
|
|
{
|
|
"e2e_ms": 20866.83848786135,
|
|
"request_id": 29,
|
|
"tpot_ms": 161.69251523328387,
|
|
"ttft_ms": 331.88905323430083
|
|
},
|
|
{
|
|
"e2e_ms": 20663.178771213603,
|
|
"request_id": 30,
|
|
"tpot_ms": 160.1348533340772,
|
|
"ttft_ms": 326.0523977857979
|
|
},
|
|
{
|
|
"e2e_ms": 20459.48769489552,
|
|
"request_id": 31,
|
|
"tpot_ms": 158.57430185728822,
|
|
"ttft_ms": 320.55135901991605
|
|
},
|
|
{
|
|
"e2e_ms": 20255.951788183756,
|
|
"request_id": 32,
|
|
"tpot_ms": 157.01541837295076,
|
|
"ttft_ms": 314.9936548190073
|
|
},
|
|
{
|
|
"e2e_ms": 20052.589009493837,
|
|
"request_id": 33,
|
|
"tpot_ms": 155.45687267778135,
|
|
"ttft_ms": 309.5661794156026
|
|
},
|
|
{
|
|
"e2e_ms": 19955.947670796646,
|
|
"request_id": 34,
|
|
"tpot_ms": 153.8983040696114,
|
|
"ttft_ms": 410.8630539559979
|
|
},
|
|
{
|
|
"e2e_ms": 19752.847929041352,
|
|
"request_id": 35,
|
|
"tpot_ms": 152.3343481355919,
|
|
"ttft_ms": 406.3857158211803
|
|
},
|
|
{
|
|
"e2e_ms": 19549.96119317403,
|
|
"request_id": 36,
|
|
"tpot_ms": 150.77184485827368,
|
|
"ttft_ms": 401.93689617327345
|
|
},
|
|
{
|
|
"e2e_ms": 19346.57831151796,
|
|
"request_id": 37,
|
|
"tpot_ms": 149.20507336304368,
|
|
"ttft_ms": 397.53399441140937
|
|
},
|
|
{
|
|
"e2e_ms": 19143.22059782458,
|
|
"request_id": 38,
|
|
"tpot_ms": 147.63442381272193,
|
|
"ttft_ms": 393.6487736088932
|
|
},
|
|
{
|
|
"e2e_ms": 18940.150721561607,
|
|
"request_id": 39,
|
|
"tpot_ms": 146.06310547094216,
|
|
"ttft_ms": 390.13632675195356
|
|
},
|
|
{
|
|
"e2e_ms": 18741.29802135198,
|
|
"request_id": 40,
|
|
"tpot_ms": 144.5231784918296,
|
|
"ttft_ms": 386.85435288961753
|
|
},
|
|
{
|
|
"e2e_ms": 18542.463082904305,
|
|
"request_id": 41,
|
|
"tpot_ms": 142.98167897793616,
|
|
"ttft_ms": 383.7898527064141
|
|
},
|
|
{
|
|
"e2e_ms": 18344.148533824395,
|
|
"request_id": 42,
|
|
"tpot_ms": 141.4401875845621,
|
|
"ttft_ms": 381.2447105850083
|
|
},
|
|
{
|
|
"e2e_ms": 18145.706224501653,
|
|
"request_id": 43,
|
|
"tpot_ms": 139.89417135289136,
|
|
"ttft_ms": 379.1464626844494
|
|
},
|
|
{
|
|
"e2e_ms": 17948.37563539231,
|
|
"request_id": 44,
|
|
"tpot_ms": 138.3533077037345,
|
|
"ttft_ms": 377.5055570180292
|
|
},
|
|
{
|
|
"e2e_ms": 17750.40475411994,
|
|
"request_id": 45,
|
|
"tpot_ms": 136.80479670989413,
|
|
"ttft_ms": 376.19557196338246
|
|
},
|
|
{
|
|
"e2e_ms": 17551.412355033975,
|
|
"request_id": 46,
|
|
"tpot_ms": 135.2471347218224,
|
|
"ttft_ms": 375.0262453625339
|
|
},
|
|
{
|
|
"e2e_ms": 17352.31516600567,
|
|
"request_id": 47,
|
|
"tpot_ms": 133.6870504253856,
|
|
"ttft_ms": 374.0597619816981
|
|
},
|
|
{
|
|
"e2e_ms": 17152.719400024922,
|
|
"request_id": 48,
|
|
"tpot_ms": 132.11931086353468,
|
|
"ttft_ms": 373.56692035601566
|
|
},
|
|
{
|
|
"e2e_ms": 16952.899525195207,
|
|
"request_id": 49,
|
|
"tpot_ms": 130.53228068835782,
|
|
"ttft_ms": 375.29987777376664
|
|
},
|
|
{
|
|
"e2e_ms": 16753.147719429508,
|
|
"request_id": 50,
|
|
"tpot_ms": 128.94354832701373,
|
|
"ttft_ms": 377.31708189876304
|
|
},
|
|
{
|
|
"e2e_ms": 16553.291092750187,
|
|
"request_id": 51,
|
|
"tpot_ms": 127.34853489391125,
|
|
"ttft_ms": 380.0271612234596
|
|
},
|
|
{
|
|
"e2e_ms": 16357.836116284518,
|
|
"request_id": 52,
|
|
"tpot_ms": 125.77762361641257,
|
|
"ttft_ms": 384.07791700011984
|
|
},
|
|
{
|
|
"e2e_ms": 16157.72333756749,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.16553823707912,
|
|
"ttft_ms": 388.6999814584407
|
|
},
|
|
{
|
|
"e2e_ms": 15952.367175996273,
|
|
"request_id": 54,
|
|
"tpot_ms": 122.51027887411597,
|
|
"ttft_ms": 393.5617589835445
|
|
},
|
|
{
|
|
"e2e_ms": 15741.765215271713,
|
|
"request_id": 55,
|
|
"tpot_ms": 120.81149800475544,
|
|
"ttft_ms": 398.70496866777216
|
|
},
|
|
{
|
|
"e2e_ms": 15528.71285895543,
|
|
"request_id": 56,
|
|
"tpot_ms": 119.08582914503211,
|
|
"ttft_ms": 404.8125575363528
|
|
},
|
|
{
|
|
"e2e_ms": 15314.765605096894,
|
|
"request_id": 57,
|
|
"tpot_ms": 117.34894161957355,
|
|
"ttft_ms": 411.4500194110526
|
|
},
|
|
{
|
|
"e2e_ms": 15096.980468100028,
|
|
"request_id": 58,
|
|
"tpot_ms": 115.58065073324867,
|
|
"ttft_ms": 418.2378249774459
|
|
},
|
|
{
|
|
"e2e_ms": 14778.441120322277,
|
|
"request_id": 59,
|
|
"tpot_ms": 113.87954551567164,
|
|
"ttft_ms": 315.7388398319796
|
|
},
|
|
{
|
|
"e2e_ms": 14557.742348852762,
|
|
"request_id": 60,
|
|
"tpot_ms": 112.08473611057838,
|
|
"ttft_ms": 322.9808628093096
|
|
},
|
|
{
|
|
"e2e_ms": 14333.83717181993,
|
|
"request_id": 61,
|
|
"tpot_ms": 110.26282336485625,
|
|
"ttft_ms": 330.4586044831872
|
|
},
|
|
{
|
|
"e2e_ms": 14108.97915094099,
|
|
"request_id": 62,
|
|
"tpot_ms": 108.43298632152924,
|
|
"ttft_ms": 337.989888106776
|
|
},
|
|
{
|
|
"e2e_ms": 13882.64463939965,
|
|
"request_id": 63,
|
|
"tpot_ms": 106.58813679106942,
|
|
"ttft_ms": 345.9512669338345
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.125,
|
|
"passed": 8
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.421875,
|
|
"passed": 27
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.71875,
|
|
"passed": 46
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "0b9f30d883de83b4af69dd114bd5d2bb974b7fd1f01d45f913bf5049e4b72a86"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 402.1317756175995,
|
|
"offered_request_rate": 3.2,
|
|
"request_metrics_sha256": "48ff1d03e0aeb45bb4395dca86e089cdbe1c7de16b8557fbe8f25e8e65c77def",
|
|
"request_rate_per_gpu": 0.4,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 26482.398681960134,
|
|
"request_id": 0,
|
|
"tpot_ms": 206.22623604112295,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 26389.27725660841,
|
|
"request_id": 1,
|
|
"tpot_ms": 204.89007083645959,
|
|
"ttft_ms": 368.23826037804076
|
|
},
|
|
{
|
|
"e2e_ms": 26186.398050214455,
|
|
"request_id": 2,
|
|
"tpot_ms": 203.43938315354237,
|
|
"ttft_ms": 349.5963897145746
|
|
},
|
|
{
|
|
"e2e_ms": 25983.46530187178,
|
|
"request_id": 3,
|
|
"tpot_ms": 201.98715121187504,
|
|
"ttft_ms": 331.0970979636474
|
|
},
|
|
{
|
|
"e2e_ms": 25780.42643937068,
|
|
"request_id": 4,
|
|
"tpot_ms": 200.53276712329495,
|
|
"ttft_ms": 312.76501471222184
|
|
},
|
|
{
|
|
"e2e_ms": 25577.34505869151,
|
|
"request_id": 5,
|
|
"tpot_ms": 199.0733214464005,
|
|
"ttft_ms": 295.0332349986446
|
|
},
|
|
{
|
|
"e2e_ms": 25483.55218185351,
|
|
"request_id": 6,
|
|
"tpot_ms": 197.69083780569105,
|
|
"ttft_ms": 376.81578053074594
|
|
},
|
|
{
|
|
"e2e_ms": 25280.28848711639,
|
|
"request_id": 7,
|
|
"tpot_ms": 196.22290463044078,
|
|
"ttft_ms": 359.97959905041466
|
|
},
|
|
{
|
|
"e2e_ms": 25076.966707259246,
|
|
"request_id": 8,
|
|
"tpot_ms": 194.75030755695653,
|
|
"ttft_ms": 343.6776475257637
|
|
},
|
|
{
|
|
"e2e_ms": 24873.46328021192,
|
|
"request_id": 9,
|
|
"tpot_ms": 193.27265662306831,
|
|
"ttft_ms": 327.83588908224147
|
|
},
|
|
{
|
|
"e2e_ms": 24669.88557698392,
|
|
"request_id": 10,
|
|
"tpot_ms": 191.78984111033463,
|
|
"ttft_ms": 312.57575597142215
|
|
},
|
|
{
|
|
"e2e_ms": 24466.212431081483,
|
|
"request_id": 11,
|
|
"tpot_ms": 190.30296300954802,
|
|
"ttft_ms": 297.7361288688827
|
|
},
|
|
{
|
|
"e2e_ms": 24370.912463364126,
|
|
"request_id": 12,
|
|
"tpot_ms": 188.79387017231738,
|
|
"ttft_ms": 394.09095147981964
|
|
},
|
|
{
|
|
"e2e_ms": 24166.86947872558,
|
|
"request_id": 13,
|
|
"tpot_ms": 187.2967209572732,
|
|
"ttft_ms": 380.1859171518851
|
|
},
|
|
{
|
|
"e2e_ms": 23962.581993212396,
|
|
"request_id": 14,
|
|
"tpot_ms": 185.79746051386414,
|
|
"ttft_ms": 366.30450795165007
|
|
},
|
|
{
|
|
"e2e_ms": 23758.13082088949,
|
|
"request_id": 15,
|
|
"tpot_ms": 184.29572160562404,
|
|
"ttft_ms": 352.57417697523687
|
|
},
|
|
{
|
|
"e2e_ms": 23552.083711717114,
|
|
"request_id": 16,
|
|
"tpot_ms": 182.7751996506789,
|
|
"ttft_ms": 339.633356080892
|
|
},
|
|
{
|
|
"e2e_ms": 23345.944189149544,
|
|
"request_id": 17,
|
|
"tpot_ms": 181.25255279330477,
|
|
"ttft_ms": 326.8699843998375
|
|
},
|
|
{
|
|
"e2e_ms": 23139.783338151214,
|
|
"request_id": 18,
|
|
"tpot_ms": 179.72694307410856,
|
|
"ttft_ms": 314.461567739424
|
|
},
|
|
{
|
|
"e2e_ms": 22933.56950157464,
|
|
"request_id": 19,
|
|
"tpot_ms": 178.1977153273412,
|
|
"ttft_ms": 302.45965500230557
|
|
},
|
|
{
|
|
"e2e_ms": 22833.726368661068,
|
|
"request_id": 20,
|
|
"tpot_ms": 176.63828124943853,
|
|
"ttft_ms": 400.6646499823745
|
|
},
|
|
{
|
|
"e2e_ms": 22627.576073166823,
|
|
"request_id": 21,
|
|
"tpot_ms": 175.09813975268048,
|
|
"ttft_ms": 390.11232457640244
|
|
},
|
|
{
|
|
"e2e_ms": 22421.391717712297,
|
|
"request_id": 22,
|
|
"tpot_ms": 173.55627478074584,
|
|
"ttft_ms": 379.7448205575735
|
|
},
|
|
{
|
|
"e2e_ms": 22215.258027879143,
|
|
"request_id": 23,
|
|
"tpot_ms": 172.01247264259936,
|
|
"ttft_ms": 369.67400226902566
|
|
},
|
|
{
|
|
"e2e_ms": 22008.978461747698,
|
|
"request_id": 24,
|
|
"tpot_ms": 170.46437149324944,
|
|
"ttft_ms": 360.0032821050201
|
|
},
|
|
{
|
|
"e2e_ms": 21802.65579951293,
|
|
"request_id": 25,
|
|
"tpot_ms": 168.91532250977505,
|
|
"ttft_ms": 350.4098407714995
|
|
},
|
|
{
|
|
"e2e_ms": 21596.434473671965,
|
|
"request_id": 26,
|
|
"tpot_ms": 167.36573728219415,
|
|
"ttft_ms": 340.98583883331025
|
|
},
|
|
{
|
|
"e2e_ms": 21390.300007810118,
|
|
"request_id": 27,
|
|
"tpot_ms": 165.81511332736412,
|
|
"ttft_ms": 331.7806152348748
|
|
},
|
|
{
|
|
"e2e_ms": 21184.224973098888,
|
|
"request_id": 28,
|
|
"tpot_ms": 164.26380332238384,
|
|
"ttft_ms": 322.72195115614187
|
|
},
|
|
{
|
|
"e2e_ms": 20978.149686441437,
|
|
"request_id": 29,
|
|
"tpot_ms": 162.71174941720716,
|
|
"ttft_ms": 313.7575104561279
|
|
},
|
|
{
|
|
"e2e_ms": 20772.153161450715,
|
|
"request_id": 30,
|
|
"tpot_ms": 161.15510654351164,
|
|
"ttft_ms": 305.4546304247392
|
|
},
|
|
{
|
|
"e2e_ms": 20672.62875180034,
|
|
"request_id": 31,
|
|
"tpot_ms": 159.59455506672822,
|
|
"ttft_ms": 404.12025832585744
|
|
},
|
|
{
|
|
"e2e_ms": 20466.69413177714,
|
|
"request_id": 32,
|
|
"tpot_ms": 158.03533915661183,
|
|
"ttft_ms": 396.2060588874419
|
|
},
|
|
{
|
|
"e2e_ms": 20260.84607576572,
|
|
"request_id": 33,
|
|
"tpot_ms": 156.475168525323,
|
|
"ttft_ms": 388.49967304969545
|
|
},
|
|
{
|
|
"e2e_ms": 20055.02118508905,
|
|
"request_id": 34,
|
|
"tpot_ms": 154.9132321630232,
|
|
"ttft_ms": 381.04070038510326
|
|
},
|
|
{
|
|
"e2e_ms": 19849.304647544788,
|
|
"request_id": 35,
|
|
"tpot_ms": 153.3461190070677,
|
|
"ttft_ms": 374.34753364718887
|
|
},
|
|
{
|
|
"e2e_ms": 19643.694612944317,
|
|
"request_id": 36,
|
|
"tpot_ms": 151.77962188577303,
|
|
"ttft_ms": 367.6826334511443
|
|
},
|
|
{
|
|
"e2e_ms": 19437.836505449406,
|
|
"request_id": 37,
|
|
"tpot_ms": 150.210341914627,
|
|
"ttft_ms": 361.1230822917744
|
|
},
|
|
{
|
|
"e2e_ms": 19231.99098193562,
|
|
"request_id": 38,
|
|
"tpot_ms": 148.63640086508497,
|
|
"ttft_ms": 355.16807206983003
|
|
},
|
|
{
|
|
"e2e_ms": 19026.289377137648,
|
|
"request_id": 39,
|
|
"tpot_ms": 147.05985988148976,
|
|
"ttft_ms": 349.6871721884496
|
|
},
|
|
{
|
|
"e2e_ms": 18822.696360365982,
|
|
"request_id": 40,
|
|
"tpot_ms": 145.49844654462956,
|
|
"ttft_ms": 344.3936491980288
|
|
},
|
|
{
|
|
"e2e_ms": 18619.112224474804,
|
|
"request_id": 41,
|
|
"tpot_ms": 143.93654065290042,
|
|
"ttft_ms": 339.17156155644966
|
|
},
|
|
{
|
|
"e2e_ms": 18415.78828326805,
|
|
"request_id": 42,
|
|
"tpot_ms": 142.37219516274803,
|
|
"ttft_ms": 334.51949759905375
|
|
},
|
|
{
|
|
"e2e_ms": 18212.400461939902,
|
|
"request_id": 43,
|
|
"tpot_ms": 140.8040960160447,
|
|
"ttft_ms": 330.280267902225
|
|
},
|
|
{
|
|
"e2e_ms": 18009.568500717836,
|
|
"request_id": 44,
|
|
"tpot_ms": 139.2366051127279,
|
|
"ttft_ms": 326.5196514013962
|
|
},
|
|
{
|
|
"e2e_ms": 17806.416393414805,
|
|
"request_id": 45,
|
|
"tpot_ms": 137.66365486442547,
|
|
"ttft_ms": 323.1322256327704
|
|
},
|
|
{
|
|
"e2e_ms": 17602.75352720573,
|
|
"request_id": 46,
|
|
"tpot_ms": 136.08599255484802,
|
|
"ttft_ms": 319.83247274003014
|
|
},
|
|
{
|
|
"e2e_ms": 17399.03826602459,
|
|
"request_id": 47,
|
|
"tpot_ms": 134.50648843613666,
|
|
"ttft_ms": 316.71423463523496
|
|
},
|
|
{
|
|
"e2e_ms": 17195.073716367715,
|
|
"request_id": 48,
|
|
"tpot_ms": 132.92201961792392,
|
|
"ttft_ms": 313.9772248913761
|
|
},
|
|
{
|
|
"e2e_ms": 16990.997112286357,
|
|
"request_id": 49,
|
|
"tpot_ms": 131.3320936883647,
|
|
"ttft_ms": 311.8212138640413
|
|
},
|
|
{
|
|
"e2e_ms": 16895.41197318666,
|
|
"request_id": 50,
|
|
"tpot_ms": 129.74336132702078,
|
|
"ttft_ms": 418.0050846550216
|
|
},
|
|
{
|
|
"e2e_ms": 16691.3169931805,
|
|
"request_id": 51,
|
|
"tpot_ms": 128.14881090811454,
|
|
"ttft_ms": 416.418007849952
|
|
},
|
|
{
|
|
"e2e_ms": 16489.422838281163,
|
|
"request_id": 52,
|
|
"tpot_ms": 126.56215930365238,
|
|
"ttft_ms": 416.0286067173118
|
|
},
|
|
{
|
|
"e2e_ms": 16285.199782255653,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.95445752714085,
|
|
"ttft_ms": 415.9836763087661
|
|
},
|
|
{
|
|
"e2e_ms": 16078.355034803544,
|
|
"request_id": 54,
|
|
"tpot_ms": 123.3249765236981,
|
|
"ttft_ms": 416.08301629388575
|
|
},
|
|
{
|
|
"e2e_ms": 15868.887387774763,
|
|
"request_id": 55,
|
|
"tpot_ms": 121.67321161824907,
|
|
"ttft_ms": 416.3895122571297
|
|
},
|
|
{
|
|
"e2e_ms": 15658.194542949623,
|
|
"request_id": 56,
|
|
"tpot_ms": 120.00563624201372,
|
|
"ttft_ms": 417.4787402138804
|
|
},
|
|
{
|
|
"e2e_ms": 15447.054249353854,
|
|
"request_id": 57,
|
|
"tpot_ms": 118.33082151865976,
|
|
"ttft_ms": 419.0399164840635
|
|
},
|
|
{
|
|
"e2e_ms": 15233.99501418892,
|
|
"request_id": 58,
|
|
"tpot_ms": 116.64063648866745,
|
|
"ttft_ms": 420.6341801281539
|
|
},
|
|
{
|
|
"e2e_ms": 15019.622333077174,
|
|
"request_id": 59,
|
|
"tpot_ms": 114.93953127109042,
|
|
"ttft_ms": 422.3018616486911
|
|
},
|
|
{
|
|
"e2e_ms": 14805.106280675915,
|
|
"request_id": 60,
|
|
"tpot_ms": 113.23514577866216,
|
|
"ttft_ms": 424.2427667858202
|
|
},
|
|
{
|
|
"e2e_ms": 14588.987025492997,
|
|
"request_id": 61,
|
|
"tpot_ms": 111.5171161341123,
|
|
"ttft_ms": 426.31327646073555
|
|
},
|
|
{
|
|
"e2e_ms": 14372.391348386529,
|
|
"request_id": 62,
|
|
"tpot_ms": 109.79194855942059,
|
|
"ttft_ms": 428.8138813401155
|
|
},
|
|
{
|
|
"e2e_ms": 14155.057425949359,
|
|
"request_id": 63,
|
|
"tpot_ms": 108.05983074662959,
|
|
"ttft_ms": 431.45892112740114
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.109375,
|
|
"passed": 7
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.40625,
|
|
"passed": 26
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.71875,
|
|
"passed": 46
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "96c4e04e8f133e66449a5567fcde568f27af44bd9f5d1cadeade66b771a8f0f3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"loads": [
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 781.3052506446838,
|
|
"offered_request_rate": 0.1,
|
|
"request_metrics_sha256": "1146954f1d73891536473385a3ecb02b14b7eaeee231291f2cbe6f9d4d222414",
|
|
"request_rate_per_gpu": 0.0125,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12579.084570749343,
|
|
"request_id": 0,
|
|
"tpot_ms": 96.75132177962064,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 12682.66705927438,
|
|
"request_id": 1,
|
|
"tpot_ms": 96.90664947750408,
|
|
"ttft_ms": 375.5225756313614
|
|
},
|
|
{
|
|
"e2e_ms": 12692.014437909165,
|
|
"request_id": 2,
|
|
"tpot_ms": 96.91832475611541,
|
|
"ttft_ms": 383.3871938825091
|
|
},
|
|
{
|
|
"e2e_ms": 12605.650216029226,
|
|
"request_id": 3,
|
|
"tpot_ms": 96.9198422551915,
|
|
"ttft_ms": 296.8302496199051
|
|
},
|
|
{
|
|
"e2e_ms": 12614.259349375281,
|
|
"request_id": 4,
|
|
"tpot_ms": 96.9125118011697,
|
|
"ttft_ms": 306.3703506267288
|
|
},
|
|
{
|
|
"e2e_ms": 12622.868482721336,
|
|
"request_id": 5,
|
|
"tpot_ms": 96.9125118011697,
|
|
"ttft_ms": 314.97948397278464
|
|
},
|
|
{
|
|
"e2e_ms": 12631.47761606615,
|
|
"request_id": 6,
|
|
"tpot_ms": 96.91251180115991,
|
|
"ttft_ms": 323.5886173188405
|
|
},
|
|
{
|
|
"e2e_ms": 12640.086749411268,
|
|
"request_id": 7,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 332.19775066687873
|
|
},
|
|
{
|
|
"e2e_ms": 12648.695882756385,
|
|
"request_id": 8,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 340.80688401199666
|
|
},
|
|
{
|
|
"e2e_ms": 12657.305016101503,
|
|
"request_id": 9,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 349.4160173571146
|
|
},
|
|
{
|
|
"e2e_ms": 12665.91414944662,
|
|
"request_id": 10,
|
|
"tpot_ms": 96.9125118011369,
|
|
"ttft_ms": 358.0251507022325
|
|
},
|
|
{
|
|
"e2e_ms": 12674.523282791739,
|
|
"request_id": 11,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 366.63428404735043
|
|
},
|
|
{
|
|
"e2e_ms": 12683.132416062932,
|
|
"request_id": 12,
|
|
"tpot_ms": 96.91251180055482,
|
|
"ttft_ms": 375.24341739246836
|
|
},
|
|
{
|
|
"e2e_ms": 12692.479794575775,
|
|
"request_id": 13,
|
|
"tpot_ms": 96.91832475507798,
|
|
"ttft_ms": 383.85255068087076
|
|
},
|
|
{
|
|
"e2e_ms": 12606.11557253071,
|
|
"request_id": 14,
|
|
"tpot_ms": 96.91984225378269,
|
|
"ttft_ms": 297.2956063003096
|
|
},
|
|
{
|
|
"e2e_ms": 12614.724705710416,
|
|
"request_id": 15,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 306.8357071486503
|
|
},
|
|
{
|
|
"e2e_ms": 12623.33383889012,
|
|
"request_id": 16,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 315.44484032835385
|
|
},
|
|
{
|
|
"e2e_ms": 12631.942972069823,
|
|
"request_id": 17,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 324.0539735080574
|
|
},
|
|
{
|
|
"e2e_ms": 12640.552105249526,
|
|
"request_id": 18,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 332.663106687761
|
|
},
|
|
{
|
|
"e2e_ms": 12649.16123842923,
|
|
"request_id": 19,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 341.2722398674646
|
|
},
|
|
{
|
|
"e2e_ms": 12657.770371608933,
|
|
"request_id": 20,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 349.88137304716815
|
|
},
|
|
{
|
|
"e2e_ms": 12666.379504788636,
|
|
"request_id": 21,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 358.49050622687173
|
|
},
|
|
{
|
|
"e2e_ms": 12674.988637968341,
|
|
"request_id": 22,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 367.0996394065753
|
|
},
|
|
{
|
|
"e2e_ms": 12683.597771148045,
|
|
"request_id": 23,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 375.7087725862789
|
|
},
|
|
{
|
|
"e2e_ms": 12692.945149660885,
|
|
"request_id": 24,
|
|
"tpot_ms": 96.91832475507798,
|
|
"ttft_ms": 384.31790576598246
|
|
},
|
|
{
|
|
"e2e_ms": 12606.580927732637,
|
|
"request_id": 25,
|
|
"tpot_ms": 96.91984225470249,
|
|
"ttft_ms": 297.7609613854213
|
|
},
|
|
{
|
|
"e2e_ms": 12615.190061213469,
|
|
"request_id": 26,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 307.3010623465393
|
|
},
|
|
{
|
|
"e2e_ms": 12623.7991946943,
|
|
"request_id": 27,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 315.9101958273709
|
|
},
|
|
{
|
|
"e2e_ms": 12632.408328175132,
|
|
"request_id": 28,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 324.5193293082025
|
|
},
|
|
{
|
|
"e2e_ms": 12641.017461655963,
|
|
"request_id": 29,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 333.1284627890341
|
|
},
|
|
{
|
|
"e2e_ms": 12649.626595136795,
|
|
"request_id": 30,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 341.73759626986566
|
|
},
|
|
{
|
|
"e2e_ms": 12658.235728617627,
|
|
"request_id": 31,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 350.34672975069725
|
|
},
|
|
{
|
|
"e2e_ms": 12666.844862098458,
|
|
"request_id": 32,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 358.95586323152884
|
|
},
|
|
{
|
|
"e2e_ms": 12675.45399557929,
|
|
"request_id": 33,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 367.5649967123604
|
|
},
|
|
{
|
|
"e2e_ms": 12684.063129060121,
|
|
"request_id": 34,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 376.174130193192
|
|
},
|
|
{
|
|
"e2e_ms": 12693.41050787034,
|
|
"request_id": 35,
|
|
"tpot_ms": 96.9183247574513,
|
|
"ttft_ms": 384.7832636740236
|
|
},
|
|
{
|
|
"e2e_ms": 12607.04628612126,
|
|
"request_id": 36,
|
|
"tpot_ms": 96.91984225613498,
|
|
"ttft_ms": 298.22631959211776
|
|
},
|
|
{
|
|
"e2e_ms": 12615.655419602092,
|
|
"request_id": 37,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 307.76642073516314
|
|
},
|
|
{
|
|
"e2e_ms": 12624.264553082923,
|
|
"request_id": 38,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 316.3755542159947
|
|
},
|
|
{
|
|
"e2e_ms": 12632.873686563755,
|
|
"request_id": 39,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 324.9846876968263
|
|
},
|
|
{
|
|
"e2e_ms": 12641.482820044586,
|
|
"request_id": 40,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 333.5938211776579
|
|
},
|
|
{
|
|
"e2e_ms": 12650.091953525418,
|
|
"request_id": 41,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 342.2029546584895
|
|
},
|
|
{
|
|
"e2e_ms": 12658.70108700625,
|
|
"request_id": 42,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 350.8120881393211
|
|
},
|
|
{
|
|
"e2e_ms": 12667.310220487081,
|
|
"request_id": 43,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 359.42122162015266
|
|
},
|
|
{
|
|
"e2e_ms": 12675.919353967913,
|
|
"request_id": 44,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 368.03035510098425
|
|
},
|
|
{
|
|
"e2e_ms": 12684.528487448744,
|
|
"request_id": 45,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 376.63948858181584
|
|
},
|
|
{
|
|
"e2e_ms": 12693.875866258964,
|
|
"request_id": 46,
|
|
"tpot_ms": 96.91832475745132,
|
|
"ttft_ms": 385.2486220626474
|
|
},
|
|
{
|
|
"e2e_ms": 12607.511644509885,
|
|
"request_id": 47,
|
|
"tpot_ms": 96.91984225613498,
|
|
"ttft_ms": 298.6916779807416
|
|
},
|
|
{
|
|
"e2e_ms": 12616.120777990716,
|
|
"request_id": 48,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 308.23177912378696
|
|
},
|
|
{
|
|
"e2e_ms": 12624.729911471548,
|
|
"request_id": 49,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 316.84091260461855
|
|
},
|
|
{
|
|
"e2e_ms": 12633.339044941862,
|
|
"request_id": 50,
|
|
"tpot_ms": 96.91251180201898,
|
|
"ttft_ms": 325.45004608545014
|
|
},
|
|
{
|
|
"e2e_ms": 12641.94817818975,
|
|
"request_id": 51,
|
|
"tpot_ms": 96.91251180018479,
|
|
"ttft_ms": 334.0591795662817
|
|
},
|
|
{
|
|
"e2e_ms": 12650.557311437638,
|
|
"request_id": 52,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 342.66831284185173
|
|
},
|
|
{
|
|
"e2e_ms": 12659.166444685525,
|
|
"request_id": 53,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 351.277446089739
|
|
},
|
|
{
|
|
"e2e_ms": 12667.775577933411,
|
|
"request_id": 54,
|
|
"tpot_ms": 96.91251179996681,
|
|
"ttft_ms": 359.88657933762624
|
|
},
|
|
{
|
|
"e2e_ms": 12676.3847111813,
|
|
"request_id": 55,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 368.4957125855135
|
|
},
|
|
{
|
|
"e2e_ms": 12684.993844429187,
|
|
"request_id": 56,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 377.10484583340076
|
|
},
|
|
{
|
|
"e2e_ms": 12694.3412230047,
|
|
"request_id": 57,
|
|
"tpot_ms": 96.91832475530246,
|
|
"ttft_ms": 385.713979081288
|
|
},
|
|
{
|
|
"e2e_ms": 12607.977001027848,
|
|
"request_id": 58,
|
|
"tpot_ms": 96.9198422540282,
|
|
"ttft_ms": 299.1570347662673
|
|
},
|
|
{
|
|
"e2e_ms": 12616.586134275734,
|
|
"request_id": 59,
|
|
"tpot_ms": 96.91251179996681,
|
|
"ttft_ms": 308.6971356799495
|
|
},
|
|
{
|
|
"e2e_ms": 12625.195267523623,
|
|
"request_id": 60,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 317.30626892783675
|
|
},
|
|
{
|
|
"e2e_ms": 12633.80440077151,
|
|
"request_id": 61,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 325.915402175724
|
|
},
|
|
{
|
|
"e2e_ms": 12642.413534019397,
|
|
"request_id": 62,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 334.52453542361127
|
|
},
|
|
{
|
|
"e2e_ms": 12436.81062947212,
|
|
"request_id": 63,
|
|
"tpot_ms": 95.22580284094978,
|
|
"ttft_ms": 343.1336686714985
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "37e166453683700c9df7abffa278431119205a29c83e92211ffa69d90e1dca0e"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 401.1116807460785,
|
|
"offered_request_rate": 0.2,
|
|
"request_metrics_sha256": "cf8a095cdf380d42316d1ad169ce9b2b45ae7d64348cd38271a332663de349ad",
|
|
"request_rate_per_gpu": 0.025,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12823.625057543633,
|
|
"request_id": 0,
|
|
"tpot_ms": 98.67683742367018,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 12920.533566322107,
|
|
"request_id": 1,
|
|
"tpot_ms": 99.10315669196875,
|
|
"ttft_ms": 334.4326664420745
|
|
},
|
|
{
|
|
"e2e_ms": 12921.724204826443,
|
|
"request_id": 2,
|
|
"tpot_ms": 99.22365133274592,
|
|
"ttft_ms": 320.32048556771286
|
|
},
|
|
{
|
|
"e2e_ms": 12922.91484333078,
|
|
"request_id": 3,
|
|
"tpot_ms": 99.22887081733072,
|
|
"ttft_ms": 320.84824952977976
|
|
},
|
|
{
|
|
"e2e_ms": 12924.105481838353,
|
|
"request_id": 4,
|
|
"tpot_ms": 99.22887081736695,
|
|
"ttft_ms": 322.0388880327505
|
|
},
|
|
{
|
|
"e2e_ms": 12925.296120352506,
|
|
"request_id": 5,
|
|
"tpot_ms": 99.22887081744425,
|
|
"ttft_ms": 323.2295265370873
|
|
},
|
|
{
|
|
"e2e_ms": 12926.486758866658,
|
|
"request_id": 6,
|
|
"tpot_ms": 99.22887081752154,
|
|
"ttft_ms": 324.420165041424
|
|
},
|
|
{
|
|
"e2e_ms": 12927.677397380812,
|
|
"request_id": 7,
|
|
"tpot_ms": 99.22887081749418,
|
|
"ttft_ms": 325.6108035590515
|
|
},
|
|
{
|
|
"e2e_ms": 12928.868035894964,
|
|
"request_id": 8,
|
|
"tpot_ms": 99.22887081749417,
|
|
"ttft_ms": 326.8014420732044
|
|
},
|
|
{
|
|
"e2e_ms": 12930.058674409118,
|
|
"request_id": 9,
|
|
"tpot_ms": 99.22887081749418,
|
|
"ttft_ms": 327.9920805873573
|
|
},
|
|
{
|
|
"e2e_ms": 12931.24931292327,
|
|
"request_id": 10,
|
|
"tpot_ms": 99.22887081749417,
|
|
"ttft_ms": 329.1827191015102
|
|
},
|
|
{
|
|
"e2e_ms": 12932.439951441467,
|
|
"request_id": 11,
|
|
"tpot_ms": 99.22887081752602,
|
|
"ttft_ms": 330.3733576156631
|
|
},
|
|
{
|
|
"e2e_ms": 12933.63058995827,
|
|
"request_id": 12,
|
|
"tpot_ms": 99.22887081754688,
|
|
"ttft_ms": 331.563996129816
|
|
},
|
|
{
|
|
"e2e_ms": 12934.821228475073,
|
|
"request_id": 13,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 332.7546346417307
|
|
},
|
|
{
|
|
"e2e_ms": 12936.011866991876,
|
|
"request_id": 14,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 333.94527315853395
|
|
},
|
|
{
|
|
"e2e_ms": 12937.20250550868,
|
|
"request_id": 15,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 335.1359116753372
|
|
},
|
|
{
|
|
"e2e_ms": 12938.393144025482,
|
|
"request_id": 16,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 336.3265501921404
|
|
},
|
|
{
|
|
"e2e_ms": 12939.583782542286,
|
|
"request_id": 17,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 337.51718870894365
|
|
},
|
|
{
|
|
"e2e_ms": 12940.77442105909,
|
|
"request_id": 18,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 338.7078272257469
|
|
},
|
|
{
|
|
"e2e_ms": 12941.965059575892,
|
|
"request_id": 19,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 339.8984657425501
|
|
},
|
|
{
|
|
"e2e_ms": 12943.155698092696,
|
|
"request_id": 20,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 341.08910425935335
|
|
},
|
|
{
|
|
"e2e_ms": 12944.346336609498,
|
|
"request_id": 21,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 342.2797427761566
|
|
},
|
|
{
|
|
"e2e_ms": 12945.536975126302,
|
|
"request_id": 22,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 343.4703812929598
|
|
},
|
|
{
|
|
"e2e_ms": 12946.727613643105,
|
|
"request_id": 23,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 344.66101980976305
|
|
},
|
|
{
|
|
"e2e_ms": 12947.918252142103,
|
|
"request_id": 24,
|
|
"tpot_ms": 99.22887081744517,
|
|
"ttft_ms": 345.8516583265663
|
|
},
|
|
{
|
|
"e2e_ms": 12949.10889064036,
|
|
"request_id": 25,
|
|
"tpot_ms": 99.22887081729912,
|
|
"ttft_ms": 347.0422968433695
|
|
},
|
|
{
|
|
"e2e_ms": 12950.299529138618,
|
|
"request_id": 26,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 348.23293534631716
|
|
},
|
|
{
|
|
"e2e_ms": 12951.490167636877,
|
|
"request_id": 27,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 349.42357384457523
|
|
},
|
|
{
|
|
"e2e_ms": 12952.680806135135,
|
|
"request_id": 28,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 350.6142123428333
|
|
},
|
|
{
|
|
"e2e_ms": 12953.871444633392,
|
|
"request_id": 29,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 351.80485084109137
|
|
},
|
|
{
|
|
"e2e_ms": 12955.06208313165,
|
|
"request_id": 30,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 352.99548933934943
|
|
},
|
|
{
|
|
"e2e_ms": 12956.25272162991,
|
|
"request_id": 31,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 354.1861278376075
|
|
},
|
|
{
|
|
"e2e_ms": 12957.443360128167,
|
|
"request_id": 32,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 355.37676633586557
|
|
},
|
|
{
|
|
"e2e_ms": 12958.633998626425,
|
|
"request_id": 33,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 356.56740483412364
|
|
},
|
|
{
|
|
"e2e_ms": 12959.824637124682,
|
|
"request_id": 34,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 357.7580433323817
|
|
},
|
|
{
|
|
"e2e_ms": 12961.015275622942,
|
|
"request_id": 35,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 358.94868183063977
|
|
},
|
|
{
|
|
"e2e_ms": 12962.2059141212,
|
|
"request_id": 36,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 360.13932032889784
|
|
},
|
|
{
|
|
"e2e_ms": 12963.396552619457,
|
|
"request_id": 37,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 361.3299588271559
|
|
},
|
|
{
|
|
"e2e_ms": 12964.587191117715,
|
|
"request_id": 38,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 362.520597325414
|
|
},
|
|
{
|
|
"e2e_ms": 12965.777829615974,
|
|
"request_id": 39,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 363.71123582367204
|
|
},
|
|
{
|
|
"e2e_ms": 12966.968468114232,
|
|
"request_id": 40,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 364.9018743219301
|
|
},
|
|
{
|
|
"e2e_ms": 12968.15910661249,
|
|
"request_id": 41,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 366.0925128201882
|
|
},
|
|
{
|
|
"e2e_ms": 12969.349745110747,
|
|
"request_id": 42,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 367.28315131844624
|
|
},
|
|
{
|
|
"e2e_ms": 12970.540383609006,
|
|
"request_id": 43,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 368.4737898167043
|
|
},
|
|
{
|
|
"e2e_ms": 12971.731022107264,
|
|
"request_id": 44,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 369.6644283149624
|
|
},
|
|
{
|
|
"e2e_ms": 12972.921660605522,
|
|
"request_id": 45,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 370.85506681322045
|
|
},
|
|
{
|
|
"e2e_ms": 12974.11229910378,
|
|
"request_id": 46,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 372.0457053114785
|
|
},
|
|
{
|
|
"e2e_ms": 12975.302937602039,
|
|
"request_id": 47,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 373.2363438097366
|
|
},
|
|
{
|
|
"e2e_ms": 12976.493576100296,
|
|
"request_id": 48,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 374.42698230799465
|
|
},
|
|
{
|
|
"e2e_ms": 12977.684214615067,
|
|
"request_id": 49,
|
|
"tpot_ms": 99.22887081739223,
|
|
"ttft_ms": 375.6176208062527
|
|
},
|
|
{
|
|
"e2e_ms": 12978.874853139188,
|
|
"request_id": 50,
|
|
"tpot_ms": 99.22887081759589,
|
|
"ttft_ms": 376.8082593045108
|
|
},
|
|
{
|
|
"e2e_ms": 12980.06549166331,
|
|
"request_id": 51,
|
|
"tpot_ms": 99.22887081779955,
|
|
"ttft_ms": 377.99889780276885
|
|
},
|
|
{
|
|
"e2e_ms": 12981.256130187432,
|
|
"request_id": 52,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 379.189536321519
|
|
},
|
|
{
|
|
"e2e_ms": 12982.446768711554,
|
|
"request_id": 53,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 380.3801748456408
|
|
},
|
|
{
|
|
"e2e_ms": 12983.637407235676,
|
|
"request_id": 54,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 381.5708133697626
|
|
},
|
|
{
|
|
"e2e_ms": 12984.828045759798,
|
|
"request_id": 55,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 382.76145189388444
|
|
},
|
|
{
|
|
"e2e_ms": 12986.01868428392,
|
|
"request_id": 56,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 383.95209041800626
|
|
},
|
|
{
|
|
"e2e_ms": 12987.209322808041,
|
|
"request_id": 57,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 385.1427289421281
|
|
},
|
|
{
|
|
"e2e_ms": 12988.399961332163,
|
|
"request_id": 58,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 386.3333674662499
|
|
},
|
|
{
|
|
"e2e_ms": 12989.881679406835,
|
|
"request_id": 59,
|
|
"tpot_ms": 99.2311627828068,
|
|
"ttft_ms": 387.52400599037173
|
|
},
|
|
{
|
|
"e2e_ms": 12991.363397481507,
|
|
"request_id": 60,
|
|
"tpot_ms": 99.23345474777176,
|
|
"ttft_ms": 388.71464451449356
|
|
},
|
|
{
|
|
"e2e_ms": 12896.836165730803,
|
|
"request_id": 61,
|
|
"tpot_ms": 99.23492287861664,
|
|
"ttft_ms": 294.0009601464908
|
|
},
|
|
{
|
|
"e2e_ms": 12689.81487892944,
|
|
"request_id": 62,
|
|
"tpot_ms": 97.59316693471084,
|
|
"ttft_ms": 295.4826782211626
|
|
},
|
|
{
|
|
"e2e_ms": 12448.118757066311,
|
|
"request_id": 63,
|
|
"tpot_ms": 95.67691266262345,
|
|
"ttft_ms": 297.1508489131338
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "e6cebfc858981b2ae4632d2c75f5e8c1f83280e7306351e15c7cd611d2664d80"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 218.6981554031372,
|
|
"offered_request_rate": 0.4,
|
|
"request_metrics_sha256": "e09d047b2039786ee02cd4c59c081ca95e5197ae65723b6170e43f6c0924921a",
|
|
"request_rate_per_gpu": 0.05,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 13556.392075191316,
|
|
"request_id": 0,
|
|
"tpot_ms": 104.4466564602661,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 13715.056219396885,
|
|
"request_id": 1,
|
|
"tpot_ms": 105.11756526634255,
|
|
"ttft_ms": 365.12543057138026
|
|
},
|
|
{
|
|
"e2e_ms": 13775.965021971426,
|
|
"request_id": 2,
|
|
"tpot_ms": 105.6336255570719,
|
|
"ttft_ms": 360.4945762232949
|
|
},
|
|
{
|
|
"e2e_ms": 13838.18765138338,
|
|
"request_id": 3,
|
|
"tpot_ms": 106.07242606904647,
|
|
"ttft_ms": 366.98954061447785
|
|
},
|
|
{
|
|
"e2e_ms": 13802.282961907104,
|
|
"request_id": 4,
|
|
"tpot_ms": 106.21712281117094,
|
|
"ttft_ms": 312.70836488839393
|
|
},
|
|
{
|
|
"e2e_ms": 13864.505591319055,
|
|
"request_id": 5,
|
|
"tpot_ms": 106.32495967758555,
|
|
"ttft_ms": 361.2357122656906
|
|
},
|
|
{
|
|
"e2e_ms": 13828.78708091747,
|
|
"request_id": 6,
|
|
"tpot_ms": 106.35892091285005,
|
|
"ttft_ms": 321.20412498551156
|
|
},
|
|
{
|
|
"e2e_ms": 13892.323156276025,
|
|
"request_id": 7,
|
|
"tpot_ms": 106.3810710779423,
|
|
"ttft_ms": 381.9271293773525
|
|
},
|
|
{
|
|
"e2e_ms": 13856.60464587857,
|
|
"request_id": 8,
|
|
"tpot_ms": 106.3914161711828,
|
|
"ttft_ms": 344.89479213835494
|
|
},
|
|
{
|
|
"e2e_ms": 13820.886135481267,
|
|
"request_id": 9,
|
|
"tpot_ms": 106.39288214818258,
|
|
"ttft_ms": 308.9901026620794
|
|
},
|
|
{
|
|
"e2e_ms": 13883.108764897528,
|
|
"request_id": 10,
|
|
"tpot_ms": 106.39288214821651,
|
|
"ttft_ms": 371.21273207403146
|
|
},
|
|
{
|
|
"e2e_ms": 13847.390254500226,
|
|
"request_id": 11,
|
|
"tpot_ms": 106.39288214825024,
|
|
"ttft_ms": 335.4942216724446
|
|
},
|
|
{
|
|
"e2e_ms": 13811.485565028235,
|
|
"request_id": 12,
|
|
"tpot_ms": 106.39288214828399,
|
|
"ttft_ms": 299.589532196169
|
|
},
|
|
{
|
|
"e2e_ms": 13873.708194444496,
|
|
"request_id": 13,
|
|
"tpot_ms": 106.38107407728566,
|
|
"ttft_ms": 363.31178662921815
|
|
},
|
|
{
|
|
"e2e_ms": 13837.989684047194,
|
|
"request_id": 14,
|
|
"tpot_ms": 106.38107407728566,
|
|
"ttft_ms": 327.5932762319158
|
|
},
|
|
{
|
|
"e2e_ms": 13901.52575941027,
|
|
"request_id": 15,
|
|
"tpot_ms": 106.39141617135505,
|
|
"ttft_ms": 389.81590564817736
|
|
},
|
|
{
|
|
"e2e_ms": 13865.807249012967,
|
|
"request_id": 16,
|
|
"tpot_ms": 106.39141617135505,
|
|
"ttft_ms": 354.097395250875
|
|
},
|
|
{
|
|
"e2e_ms": 13830.088738615665,
|
|
"request_id": 17,
|
|
"tpot_ms": 106.3928821483211,
|
|
"ttft_ms": 318.19270577888403
|
|
},
|
|
{
|
|
"e2e_ms": 13893.62481397874,
|
|
"request_id": 18,
|
|
"tpot_ms": 106.40322424239051,
|
|
"ttft_ms": 380.4153351951456
|
|
},
|
|
{
|
|
"e2e_ms": 13857.906303581438,
|
|
"request_id": 19,
|
|
"tpot_ms": 106.40322424239051,
|
|
"ttft_ms": 344.69682479784325
|
|
},
|
|
{
|
|
"e2e_ms": 13822.187793184135,
|
|
"request_id": 20,
|
|
"tpot_ms": 106.40469021935657,
|
|
"ttft_ms": 308.79213532585226
|
|
},
|
|
{
|
|
"e2e_ms": 13884.410422593844,
|
|
"request_id": 21,
|
|
"tpot_ms": 106.39288214826952,
|
|
"ttft_ms": 372.51438976361584
|
|
},
|
|
{
|
|
"e2e_ms": 13848.691912190276,
|
|
"request_id": 22,
|
|
"tpot_ms": 106.39288214822018,
|
|
"ttft_ms": 336.7958793663135
|
|
},
|
|
{
|
|
"e2e_ms": 13812.78722271334,
|
|
"request_id": 23,
|
|
"tpot_ms": 106.39288214818123,
|
|
"ttft_ms": 300.8911898943225
|
|
},
|
|
{
|
|
"e2e_ms": 13875.009852123696,
|
|
"request_id": 24,
|
|
"tpot_ms": 106.38107407709929,
|
|
"ttft_ms": 364.6134443320861
|
|
},
|
|
{
|
|
"e2e_ms": 13839.291341720127,
|
|
"request_id": 25,
|
|
"tpot_ms": 106.38107407704995,
|
|
"ttft_ms": 328.89493393478375
|
|
},
|
|
{
|
|
"e2e_ms": 13902.827417078257,
|
|
"request_id": 26,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 391.11756334850156
|
|
},
|
|
{
|
|
"e2e_ms": 13867.108906674688,
|
|
"request_id": 27,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 355.39905294493224
|
|
},
|
|
{
|
|
"e2e_ms": 13831.390396271117,
|
|
"request_id": 28,
|
|
"tpot_ms": 106.39288214805607,
|
|
"ttft_ms": 319.49436346799587
|
|
},
|
|
{
|
|
"e2e_ms": 13894.926471629247,
|
|
"request_id": 29,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 381.7169928783528
|
|
},
|
|
{
|
|
"e2e_ms": 13859.207961225678,
|
|
"request_id": 30,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 345.9984824747835
|
|
},
|
|
{
|
|
"e2e_ms": 13823.48945082211,
|
|
"request_id": 31,
|
|
"tpot_ms": 106.40469021908869,
|
|
"ttft_ms": 310.0937929978471
|
|
},
|
|
{
|
|
"e2e_ms": 13885.712080232466,
|
|
"request_id": 32,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 373.8160474293437
|
|
},
|
|
{
|
|
"e2e_ms": 13849.993569828897,
|
|
"request_id": 33,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 338.0975370257744
|
|
},
|
|
{
|
|
"e2e_ms": 13814.08888035196,
|
|
"request_id": 34,
|
|
"tpot_ms": 106.39288214805607,
|
|
"ttft_ms": 302.192847548838
|
|
},
|
|
{
|
|
"e2e_ms": 13876.311509762318,
|
|
"request_id": 35,
|
|
"tpot_ms": 106.3810740770235,
|
|
"ttft_ms": 365.9151019803346
|
|
},
|
|
{
|
|
"e2e_ms": 13840.592999358747,
|
|
"request_id": 36,
|
|
"tpot_ms": 106.38107407702348,
|
|
"ttft_ms": 330.1965915767653
|
|
},
|
|
{
|
|
"e2e_ms": 13904.129074716877,
|
|
"request_id": 37,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 392.41922098712223
|
|
},
|
|
{
|
|
"e2e_ms": 13868.410564313308,
|
|
"request_id": 38,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 356.7007105835529
|
|
},
|
|
{
|
|
"e2e_ms": 13832.69205390974,
|
|
"request_id": 39,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 320.79602110661654
|
|
},
|
|
{
|
|
"e2e_ms": 13896.228129267869,
|
|
"request_id": 40,
|
|
"tpot_ms": 106.40322424213304,
|
|
"ttft_ms": 383.0186505169735
|
|
},
|
|
{
|
|
"e2e_ms": 13860.509618864298,
|
|
"request_id": 41,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 347.30014011340415
|
|
},
|
|
{
|
|
"e2e_ms": 13824.79110846073,
|
|
"request_id": 42,
|
|
"tpot_ms": 106.40469021908868,
|
|
"ttft_ms": 311.3954506364678
|
|
},
|
|
{
|
|
"e2e_ms": 13888.32718381886,
|
|
"request_id": 43,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 375.1177050679644
|
|
},
|
|
{
|
|
"e2e_ms": 13852.60867341529,
|
|
"request_id": 44,
|
|
"tpot_ms": 106.40322424213304,
|
|
"ttft_ms": 339.39919466439505
|
|
},
|
|
{
|
|
"e2e_ms": 13816.89016301172,
|
|
"request_id": 45,
|
|
"tpot_ms": 106.40469021908868,
|
|
"ttft_ms": 303.4945051874587
|
|
},
|
|
{
|
|
"e2e_ms": 13879.11279242067,
|
|
"request_id": 46,
|
|
"tpot_ms": 106.392882148045,
|
|
"ttft_ms": 367.2167596189553
|
|
},
|
|
{
|
|
"e2e_ms": 13843.394282013549,
|
|
"request_id": 47,
|
|
"tpot_ms": 106.39288214801702,
|
|
"ttft_ms": 331.49824921538595
|
|
},
|
|
{
|
|
"e2e_ms": 13807.489592530403,
|
|
"request_id": 48,
|
|
"tpot_ms": 106.39288214796812,
|
|
"ttft_ms": 295.5935597384496
|
|
},
|
|
{
|
|
"e2e_ms": 13869.712221935884,
|
|
"request_id": 49,
|
|
"tpot_ms": 106.38107407689715,
|
|
"ttft_ms": 359.3158141699462
|
|
},
|
|
{
|
|
"e2e_ms": 13833.993711528763,
|
|
"request_id": 50,
|
|
"tpot_ms": 106.38107407686918,
|
|
"ttft_ms": 323.59730376637685
|
|
},
|
|
{
|
|
"e2e_ms": 13897.5297868798,
|
|
"request_id": 51,
|
|
"tpot_ms": 106.3914161708903,
|
|
"ttft_ms": 385.8199331767338
|
|
},
|
|
{
|
|
"e2e_ms": 13861.81127647268,
|
|
"request_id": 52,
|
|
"tpot_ms": 106.39141617089378,
|
|
"ttft_ms": 350.1014227691712
|
|
},
|
|
{
|
|
"e2e_ms": 13826.092766065556,
|
|
"request_id": 53,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 314.1967332860247
|
|
},
|
|
{
|
|
"e2e_ms": 13889.628841416596,
|
|
"request_id": 54,
|
|
"tpot_ms": 106.40322424192983,
|
|
"ttft_ms": 376.41936269150733
|
|
},
|
|
{
|
|
"e2e_ms": 13853.910331009472,
|
|
"request_id": 55,
|
|
"tpot_ms": 106.4032242419298,
|
|
"ttft_ms": 340.7008522843853
|
|
},
|
|
{
|
|
"e2e_ms": 13818.191820602351,
|
|
"request_id": 56,
|
|
"tpot_ms": 106.4046902189064,
|
|
"ttft_ms": 304.7961628012388
|
|
},
|
|
{
|
|
"e2e_ms": 13880.414450007833,
|
|
"request_id": 57,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 368.5184172283016
|
|
},
|
|
{
|
|
"e2e_ms": 13844.695939600711,
|
|
"request_id": 58,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 332.79990682117955
|
|
},
|
|
{
|
|
"e2e_ms": 13600.134841430987,
|
|
"request_id": 59,
|
|
"tpot_ms": 104.74991829994453,
|
|
"ttft_ms": 296.89521733803304
|
|
},
|
|
{
|
|
"e2e_ms": 13451.749583806417,
|
|
"request_id": 60,
|
|
"tpot_ms": 103.07978040977419,
|
|
"ttft_ms": 360.61747176509584
|
|
},
|
|
{
|
|
"e2e_ms": 13168.506714577745,
|
|
"request_id": 61,
|
|
"tpot_ms": 101.13076971039189,
|
|
"ttft_ms": 324.8989613579738
|
|
},
|
|
{
|
|
"e2e_ms": 12970.21046401943,
|
|
"request_id": 62,
|
|
"tpot_ms": 99.07943994689744,
|
|
"ttft_ms": 387.1215907634564
|
|
},
|
|
{
|
|
"e2e_ms": 12659.030247905519,
|
|
"request_id": 63,
|
|
"tpot_ms": 96.91045013818255,
|
|
"ttft_ms": 351.4030803563344
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "021442528b6c4067d63827496b66b6f530adbe9f4b29732b1aa7aab2112ced9f"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 144.67649912834167,
|
|
"offered_request_rate": 0.8,
|
|
"request_metrics_sha256": "6b246d56546536e46b4e8adfb23076b3cbf053da4b0af0288346df10122d6022",
|
|
"request_rate_per_gpu": 0.1,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 15423.90313583537,
|
|
"request_id": 0,
|
|
"tpot_ms": 119.15146796140039,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 15680.782114004433,
|
|
"request_id": 1,
|
|
"tpot_ms": 120.47488426274414,
|
|
"ttft_ms": 380.47181263592745
|
|
},
|
|
{
|
|
"e2e_ms": 15824.99160915063,
|
|
"request_id": 2,
|
|
"tpot_ms": 121.60222806639713,
|
|
"ttft_ms": 381.5086447181937
|
|
},
|
|
{
|
|
"e2e_ms": 15964.925902693089,
|
|
"request_id": 3,
|
|
"tpot_ms": 122.6571852387084,
|
|
"ttft_ms": 387.4633773771219
|
|
},
|
|
{
|
|
"e2e_ms": 15994.3969757696,
|
|
"request_id": 4,
|
|
"tpot_ms": 123.48492728290344,
|
|
"ttft_ms": 311.8112108408626
|
|
},
|
|
{
|
|
"e2e_ms": 16132.273068839513,
|
|
"request_id": 5,
|
|
"tpot_ms": 124.38017511477935,
|
|
"ttft_ms": 335.9908292625358
|
|
},
|
|
{
|
|
"e2e_ms": 16268.09096143688,
|
|
"request_id": 6,
|
|
"tpot_ms": 125.15372351796954,
|
|
"ttft_ms": 373.5680746547478
|
|
},
|
|
{
|
|
"e2e_ms": 16291.239371010352,
|
|
"request_id": 7,
|
|
"tpot_ms": 125.67763083904141,
|
|
"ttft_ms": 330.18025445209173
|
|
},
|
|
{
|
|
"e2e_ms": 16612.993923996273,
|
|
"request_id": 8,
|
|
"tpot_ms": 127.71634872045071,
|
|
"ttft_ms": 393.0176364990334
|
|
},
|
|
{
|
|
"e2e_ms": 16636.356874836954,
|
|
"request_id": 9,
|
|
"tpot_ms": 128.11558123152213,
|
|
"ttft_ms": 365.67805843364454
|
|
},
|
|
{
|
|
"e2e_ms": 16659.624174048608,
|
|
"request_id": 10,
|
|
"tpot_ms": 128.3236060799542,
|
|
"ttft_ms": 362.52620189442376
|
|
},
|
|
{
|
|
"e2e_ms": 16683.03953534565,
|
|
"request_id": 11,
|
|
"tpot_ms": 128.34365422438844,
|
|
"ttft_ms": 383.3954488483169
|
|
},
|
|
{
|
|
"e2e_ms": 16598.14552827833,
|
|
"request_id": 12,
|
|
"tpot_ms": 128.20865630197483,
|
|
"ttft_ms": 315.6461779275297
|
|
},
|
|
{
|
|
"e2e_ms": 16621.508479120537,
|
|
"request_id": 13,
|
|
"tpot_ms": 128.0747814081899,
|
|
"ttft_ms": 356.0112402804201
|
|
},
|
|
{
|
|
"e2e_ms": 16644.871429963147,
|
|
"request_id": 14,
|
|
"tpot_ms": 127.97565080664988,
|
|
"ttft_ms": 391.9637775186118
|
|
},
|
|
{
|
|
"e2e_ms": 16559.977422897646,
|
|
"request_id": 15,
|
|
"tpot_ms": 127.91134905305496,
|
|
"ttft_ms": 315.2360931596654
|
|
},
|
|
{
|
|
"e2e_ms": 16583.340373740255,
|
|
"request_id": 16,
|
|
"tpot_ms": 127.86325360239432,
|
|
"ttft_ms": 344.7071662361765
|
|
},
|
|
{
|
|
"e2e_ms": 16606.703324582868,
|
|
"request_id": 17,
|
|
"tpot_ms": 127.81515815173371,
|
|
"ttft_ms": 374.17823931268754
|
|
},
|
|
{
|
|
"e2e_ms": 16630.066275425477,
|
|
"request_id": 18,
|
|
"tpot_ms": 127.7832690040065,
|
|
"ttft_ms": 401.59111191665176
|
|
},
|
|
{
|
|
"e2e_ms": 16354.971144941388,
|
|
"request_id": 19,
|
|
"tpot_ms": 126.2873116642462,
|
|
"ttft_ms": 316.48256358212024
|
|
},
|
|
{
|
|
"e2e_ms": 16568.46874002116,
|
|
"request_id": 20,
|
|
"tpot_ms": 127.78612414854776,
|
|
"ttft_ms": 339.6309731555931
|
|
},
|
|
{
|
|
"e2e_ms": 16591.83169086377,
|
|
"request_id": 21,
|
|
"tpot_ms": 127.78612414856296,
|
|
"ttft_ms": 362.9939239962745
|
|
},
|
|
{
|
|
"e2e_ms": 16615.19464170638,
|
|
"request_id": 22,
|
|
"tpot_ms": 127.78612414857815,
|
|
"ttft_ms": 386.3568748369559
|
|
},
|
|
{
|
|
"e2e_ms": 16339.951449136564,
|
|
"request_id": 23,
|
|
"tpot_ms": 126.28847750749865,
|
|
"ttft_ms": 301.3148056842354
|
|
},
|
|
{
|
|
"e2e_ms": 16553.449044216337,
|
|
"request_id": 24,
|
|
"tpot_ms": 127.78518800972483,
|
|
"ttft_ms": 324.73016698128276
|
|
},
|
|
{
|
|
"e2e_ms": 16576.811995058946,
|
|
"request_id": 25,
|
|
"tpot_ms": 127.78477532898123,
|
|
"ttft_ms": 348.1455282783301
|
|
},
|
|
{
|
|
"e2e_ms": 16600.174945901555,
|
|
"request_id": 26,
|
|
"tpot_ms": 127.78477532898441,
|
|
"ttft_ms": 371.50847912053564
|
|
},
|
|
{
|
|
"e2e_ms": 16623.53789674417,
|
|
"request_id": 27,
|
|
"tpot_ms": 127.78477532898442,
|
|
"ttft_ms": 394.87142996314617
|
|
},
|
|
{
|
|
"e2e_ms": 16348.29470417435,
|
|
"request_id": 28,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 309.97742289764574
|
|
},
|
|
{
|
|
"e2e_ms": 16561.79229925412,
|
|
"request_id": 29,
|
|
"tpot_ms": 127.78308602766822,
|
|
"ttft_ms": 333.34037374025627
|
|
},
|
|
{
|
|
"e2e_ms": 16585.155250096734,
|
|
"request_id": 30,
|
|
"tpot_ms": 127.78308602766825,
|
|
"ttft_ms": 356.7033245828668
|
|
},
|
|
{
|
|
"e2e_ms": 16608.518200939343,
|
|
"request_id": 31,
|
|
"tpot_ms": 127.78308602766823,
|
|
"ttft_ms": 380.06627542547733
|
|
},
|
|
{
|
|
"e2e_ms": 16631.881151781956,
|
|
"request_id": 32,
|
|
"tpot_ms": 127.78360948578973,
|
|
"ttft_ms": 403.36274708666053
|
|
},
|
|
{
|
|
"e2e_ms": 16356.786021297865,
|
|
"request_id": 33,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 318.4687400211601
|
|
},
|
|
{
|
|
"e2e_ms": 16570.283616377637,
|
|
"request_id": 34,
|
|
"tpot_ms": 127.78308602766825,
|
|
"ttft_ms": 341.83169086377063
|
|
},
|
|
{
|
|
"e2e_ms": 16593.646567220247,
|
|
"request_id": 35,
|
|
"tpot_ms": 127.78308602766823,
|
|
"ttft_ms": 365.19464170638116
|
|
},
|
|
{
|
|
"e2e_ms": 16617.00951806286,
|
|
"request_id": 36,
|
|
"tpot_ms": 127.78477532898444,
|
|
"ttft_ms": 388.34305128183644
|
|
},
|
|
{
|
|
"e2e_ms": 16341.766325493041,
|
|
"request_id": 37,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 303.449044216336
|
|
},
|
|
{
|
|
"e2e_ms": 16555.263920572714,
|
|
"request_id": 38,
|
|
"tpot_ms": 127.78308602766747,
|
|
"ttft_ms": 326.81199505894654
|
|
},
|
|
{
|
|
"e2e_ms": 16578.62687141207,
|
|
"request_id": 39,
|
|
"tpot_ms": 127.78308602764184,
|
|
"ttft_ms": 350.17494590155707
|
|
},
|
|
{
|
|
"e2e_ms": 16601.989822251424,
|
|
"request_id": 40,
|
|
"tpot_ms": 127.78308602761618,
|
|
"ttft_ms": 373.5378967441676
|
|
},
|
|
{
|
|
"e2e_ms": 16625.35277309078,
|
|
"request_id": 41,
|
|
"tpot_ms": 127.78477532890675,
|
|
"ttft_ms": 396.6863063196229
|
|
},
|
|
{
|
|
"e2e_ms": 16350.109580518321,
|
|
"request_id": 42,
|
|
"tpot_ms": 126.2859628446,
|
|
"ttft_ms": 311.79229925412244
|
|
},
|
|
{
|
|
"e2e_ms": 16563.607175594883,
|
|
"request_id": 43,
|
|
"tpot_ms": 127.78308602754448,
|
|
"ttft_ms": 335.15525009673297
|
|
},
|
|
{
|
|
"e2e_ms": 16586.970126434237,
|
|
"request_id": 44,
|
|
"tpot_ms": 127.78308602751885,
|
|
"ttft_ms": 358.5182009393435
|
|
},
|
|
{
|
|
"e2e_ms": 16610.333077273594,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.78308602749323,
|
|
"ttft_ms": 381.88115178195403
|
|
},
|
|
{
|
|
"e2e_ms": 16633.69602811295,
|
|
"request_id": 46,
|
|
"tpot_ms": 127.78360948558907,
|
|
"ttft_ms": 405.17762344313724
|
|
},
|
|
{
|
|
"e2e_ms": 16548.802021044266,
|
|
"request_id": 47,
|
|
"tpot_ms": 127.78360948556401,
|
|
"ttft_ms": 320.2836163776368
|
|
},
|
|
{
|
|
"e2e_ms": 16572.164971883623,
|
|
"request_id": 48,
|
|
"tpot_ms": 127.7836094855384,
|
|
"ttft_ms": 343.64656722024733
|
|
},
|
|
{
|
|
"e2e_ms": 16595.52792272298,
|
|
"request_id": 49,
|
|
"tpot_ms": 127.78360948551277,
|
|
"ttft_ms": 367.00951806285786
|
|
},
|
|
{
|
|
"e2e_ms": 16618.890873562337,
|
|
"request_id": 50,
|
|
"tpot_ms": 127.78529878680334,
|
|
"ttft_ms": 390.15792763831314
|
|
},
|
|
{
|
|
"e2e_ms": 16343.647680989874,
|
|
"request_id": 51,
|
|
"tpot_ms": 126.28648630249732,
|
|
"ttft_ms": 305.26392057271323
|
|
},
|
|
{
|
|
"e2e_ms": 16199.563507361632,
|
|
"request_id": 52,
|
|
"tpot_ms": 124.96800500747688,
|
|
"ttft_ms": 328.6268714120695
|
|
},
|
|
{
|
|
"e2e_ms": 16032.332947106512,
|
|
"request_id": 53,
|
|
"tpot_ms": 123.46726869964634,
|
|
"ttft_ms": 351.9898222514257
|
|
},
|
|
{
|
|
"e2e_ms": 15838.885472588829,
|
|
"request_id": 54,
|
|
"tpot_ms": 121.7600999960476,
|
|
"ttft_ms": 375.35277309078197
|
|
},
|
|
{
|
|
"e2e_ms": 15619.209002299955,
|
|
"request_id": 55,
|
|
"tpot_ms": 119.84809306800307,
|
|
"ttft_ms": 398.5011826635656
|
|
},
|
|
{
|
|
"e2e_ms": 15285.473398871076,
|
|
"request_id": 56,
|
|
"tpot_ms": 117.8887104194976,
|
|
"ttft_ms": 313.60717559488194
|
|
},
|
|
{
|
|
"e2e_ms": 15049.070462905931,
|
|
"request_id": 57,
|
|
"tpot_ms": 115.84330973599758,
|
|
"ttft_ms": 336.9701264342382
|
|
},
|
|
{
|
|
"e2e_ms": 14793.47811125659,
|
|
"request_id": 58,
|
|
"tpot_ms": 113.6468112912047,
|
|
"ttft_ms": 360.3330772735944
|
|
},
|
|
{
|
|
"e2e_ms": 14524.751300129523,
|
|
"request_id": 59,
|
|
"tpot_ms": 111.34689190564231,
|
|
"ttft_ms": 383.69602811295067
|
|
},
|
|
{
|
|
"e2e_ms": 14156.606828520311,
|
|
"request_id": 60,
|
|
"tpot_ms": 109.11657328721294,
|
|
"ttft_ms": 298.802021044267
|
|
},
|
|
{
|
|
"e2e_ms": 13870.414276689999,
|
|
"request_id": 61,
|
|
"tpot_ms": 106.67912838430216,
|
|
"ttft_ms": 322.16497188362325
|
|
},
|
|
{
|
|
"e2e_ms": 13579.457505623892,
|
|
"request_id": 62,
|
|
"tpot_ms": 104.20416994410166,
|
|
"ttft_ms": 345.5279227229795
|
|
},
|
|
{
|
|
"e2e_ms": 13281.118281252404,
|
|
"request_id": 63,
|
|
"tpot_ms": 101.6710819503155,
|
|
"ttft_ms": 368.89087356233574
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.15625,
|
|
"passed": 10
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "9ed2bf69d3193253248833822a357db5130e6ca29282e93e04cc92961dba468d"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 138.42228722572327,
|
|
"offered_request_rate": 1.2,
|
|
"request_metrics_sha256": "dfafc592cb8f7c26725458265ca04cb89642135218d5f80993dfbd6e83fcea9c",
|
|
"request_rate_per_gpu": 0.15,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 17349.6420929535,
|
|
"request_id": 0,
|
|
"tpot_ms": 134.3147668363463,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 17472.38469413562,
|
|
"request_id": 1,
|
|
"tpot_ms": 135.04025977135157,
|
|
"ttft_ms": 322.27170317397037
|
|
},
|
|
{
|
|
"e2e_ms": 17594.570506858923,
|
|
"request_id": 2,
|
|
"tpot_ms": 135.72808492940857,
|
|
"ttft_ms": 357.1037208240346
|
|
},
|
|
{
|
|
"e2e_ms": 17607.2367009099,
|
|
"request_id": 3,
|
|
"tpot_ms": 136.28731175562888,
|
|
"ttft_ms": 298.7481079450336
|
|
},
|
|
{
|
|
"e2e_ms": 17921.914449676293,
|
|
"request_id": 4,
|
|
"tpot_ms": 138.39011926907332,
|
|
"ttft_ms": 346.36930250398024
|
|
},
|
|
{
|
|
"e2e_ms": 17935.406822135243,
|
|
"request_id": 5,
|
|
"tpot_ms": 138.88107821741167,
|
|
"ttft_ms": 297.50988852396
|
|
},
|
|
{
|
|
"e2e_ms": 18057.925904932094,
|
|
"request_id": 6,
|
|
"tpot_ms": 139.39521236971183,
|
|
"ttft_ms": 354.73393397869125
|
|
},
|
|
{
|
|
"e2e_ms": 18071.482157513437,
|
|
"request_id": 7,
|
|
"tpot_ms": 139.75067177492934,
|
|
"ttft_ms": 323.14684209741085
|
|
},
|
|
{
|
|
"e2e_ms": 18194.065120430685,
|
|
"request_id": 8,
|
|
"tpot_ms": 140.142283173617,
|
|
"ttft_ms": 395.99515738132584
|
|
},
|
|
{
|
|
"e2e_ms": 18207.834601589766,
|
|
"request_id": 9,
|
|
"tpot_ms": 140.4257392143267,
|
|
"ttft_ms": 373.76572137027256
|
|
},
|
|
{
|
|
"e2e_ms": 18221.607066695313,
|
|
"request_id": 10,
|
|
"tpot_ms": 140.60137526077196,
|
|
"ttft_ms": 365.232408577274
|
|
},
|
|
{
|
|
"e2e_ms": 18235.379531799863,
|
|
"request_id": 11,
|
|
"tpot_ms": 140.670460527253,
|
|
"ttft_ms": 370.23104483873226
|
|
},
|
|
{
|
|
"e2e_ms": 18249.21289308034,
|
|
"request_id": 12,
|
|
"tpot_ms": 140.64257855984852,
|
|
"ttft_ms": 387.6054159795803
|
|
},
|
|
{
|
|
"e2e_ms": 18263.046254360816,
|
|
"request_id": 13,
|
|
"tpot_ms": 140.69995603545655,
|
|
"ttft_ms": 394.1518378578337
|
|
},
|
|
{
|
|
"e2e_ms": 18276.879615640297,
|
|
"request_id": 14,
|
|
"tpot_ms": 140.75508408264292,
|
|
"ttft_ms": 400.98393714464817
|
|
},
|
|
{
|
|
"e2e_ms": 18181.536918127527,
|
|
"request_id": 15,
|
|
"tpot_ms": 140.80327015717634,
|
|
"ttft_ms": 299.52160816613116
|
|
},
|
|
{
|
|
"e2e_ms": 18195.15705083027,
|
|
"request_id": 16,
|
|
"tpot_ms": 140.84624138271937,
|
|
"ttft_ms": 307.6843952249071
|
|
},
|
|
{
|
|
"e2e_ms": 18208.77718353195,
|
|
"request_id": 17,
|
|
"tpot_ms": 140.87800086343316,
|
|
"ttft_ms": 317.2710738759417
|
|
},
|
|
{
|
|
"e2e_ms": 18222.54666469151,
|
|
"request_id": 18,
|
|
"tpot_ms": 140.90607860065492,
|
|
"ttft_ms": 327.4746824083348
|
|
},
|
|
{
|
|
"e2e_ms": 18236.319129797193,
|
|
"request_id": 19,
|
|
"tpot_ms": 140.91088449209656,
|
|
"ttft_ms": 340.6367993009294
|
|
},
|
|
{
|
|
"e2e_ms": 18250.152491077173,
|
|
"request_id": 20,
|
|
"tpot_ms": 140.8981539798204,
|
|
"ttft_ms": 356.0869356399827
|
|
},
|
|
{
|
|
"e2e_ms": 18263.985852358146,
|
|
"request_id": 21,
|
|
"tpot_ms": 140.89857595079965,
|
|
"ttft_ms": 369.8667066065901
|
|
},
|
|
{
|
|
"e2e_ms": 18277.81921363912,
|
|
"request_id": 22,
|
|
"tpot_ms": 140.90338208286406,
|
|
"ttft_ms": 383.0896891153834
|
|
},
|
|
{
|
|
"e2e_ms": 18291.6525749191,
|
|
"request_id": 23,
|
|
"tpot_ms": 140.91257237601366,
|
|
"ttft_ms": 395.7558831653678
|
|
},
|
|
{
|
|
"e2e_ms": 18305.48593620007,
|
|
"request_id": 24,
|
|
"tpot_ms": 140.9217626691632,
|
|
"ttft_ms": 408.4220772163469
|
|
},
|
|
{
|
|
"e2e_ms": 18210.143238687502,
|
|
"request_id": 25,
|
|
"tpot_ms": 140.92327164841032,
|
|
"ttft_ms": 312.8877393393914
|
|
},
|
|
{
|
|
"e2e_ms": 18223.912719846063,
|
|
"request_id": 26,
|
|
"tpot_ms": 140.92545360667498,
|
|
"ttft_ms": 326.38011179834336
|
|
},
|
|
{
|
|
"e2e_ms": 18237.68518495175,
|
|
"request_id": 27,
|
|
"tpot_ms": 140.92765906057843,
|
|
"ttft_ms": 339.8724842582901
|
|
},
|
|
{
|
|
"e2e_ms": 18251.518546232724,
|
|
"request_id": 28,
|
|
"tpot_ms": 140.92984101884323,
|
|
"ttft_ms": 353.42873683963205
|
|
},
|
|
{
|
|
"e2e_ms": 18265.3519075127,
|
|
"request_id": 29,
|
|
"tpot_ms": 140.93202297710803,
|
|
"ttft_ms": 366.98498941997923
|
|
},
|
|
{
|
|
"e2e_ms": 18279.185268793677,
|
|
"request_id": 30,
|
|
"tpot_ms": 140.93370194229104,
|
|
"ttft_ms": 380.60512212271647
|
|
},
|
|
{
|
|
"e2e_ms": 18293.01863007465,
|
|
"request_id": 31,
|
|
"tpot_ms": 140.9341814397353,
|
|
"ttft_ms": 394.3775872282629
|
|
},
|
|
{
|
|
"e2e_ms": 18306.851991354626,
|
|
"request_id": 32,
|
|
"tpot_ms": 140.93466093717961,
|
|
"ttft_ms": 408.15005233281454
|
|
},
|
|
{
|
|
"e2e_ms": 18211.509293842057,
|
|
"request_id": 33,
|
|
"tpot_ms": 140.933484965084,
|
|
"ttft_ms": 312.956703276388
|
|
},
|
|
{
|
|
"e2e_ms": 18225.278775001614,
|
|
"request_id": 34,
|
|
"tpot_ms": 140.9329819720059,
|
|
"ttft_ms": 326.79006455686374
|
|
},
|
|
{
|
|
"e2e_ms": 18239.11213628159,
|
|
"request_id": 35,
|
|
"tpot_ms": 140.93298197200983,
|
|
"ttft_ms": 340.6234258363447
|
|
},
|
|
{
|
|
"e2e_ms": 18252.945497562563,
|
|
"request_id": 36,
|
|
"tpot_ms": 140.93298197201372,
|
|
"ttft_ms": 354.45678711682046
|
|
},
|
|
{
|
|
"e2e_ms": 18266.778858843536,
|
|
"request_id": 37,
|
|
"tpot_ms": 140.93466093719667,
|
|
"ttft_ms": 368.0769198195577
|
|
},
|
|
{
|
|
"e2e_ms": 18280.612220123516,
|
|
"request_id": 38,
|
|
"tpot_ms": 140.93633990237933,
|
|
"ttft_ms": 381.69705252133923
|
|
},
|
|
{
|
|
"e2e_ms": 18294.44558140449,
|
|
"request_id": 39,
|
|
"tpot_ms": 140.93801886756114,
|
|
"ttft_ms": 395.3171852242221
|
|
},
|
|
{
|
|
"e2e_ms": 18308.27894268546,
|
|
"request_id": 40,
|
|
"tpot_ms": 140.93849836500434,
|
|
"ttft_ms": 409.0896503299106
|
|
},
|
|
{
|
|
"e2e_ms": 18212.936245171895,
|
|
"request_id": 41,
|
|
"tpot_ms": 140.93732239290452,
|
|
"ttft_ms": 313.8963012730187
|
|
},
|
|
{
|
|
"e2e_ms": 18226.708710277584,
|
|
"request_id": 42,
|
|
"tpot_ms": 140.93684289546135,
|
|
"ttft_ms": 327.72966255399183
|
|
},
|
|
{
|
|
"e2e_ms": 18047.749752064,
|
|
"request_id": 43,
|
|
"tpot_ms": 139.41879313566167,
|
|
"ttft_ms": 341.56302383496495
|
|
},
|
|
{
|
|
"e2e_ms": 17872.42465139999,
|
|
"request_id": 44,
|
|
"tpot_ms": 137.92935642744132,
|
|
"ttft_ms": 355.3963851149433
|
|
},
|
|
{
|
|
"e2e_ms": 17695.178674250878,
|
|
"request_id": 45,
|
|
"tpot_ms": 136.4247947075194,
|
|
"ttft_ms": 369.22974639591644
|
|
},
|
|
{
|
|
"e2e_ms": 17514.86814366097,
|
|
"request_id": 46,
|
|
"tpot_ms": 134.89610264554392,
|
|
"ttft_ms": 383.06310767688956
|
|
},
|
|
{
|
|
"e2e_ms": 17334.24324324006,
|
|
"request_id": 47,
|
|
"tpot_ms": 133.36661419575813,
|
|
"ttft_ms": 396.68324037877767
|
|
},
|
|
{
|
|
"e2e_ms": 17152.122611965817,
|
|
"request_id": 48,
|
|
"tpot_ms": 131.8241488699319,
|
|
"ttft_ms": 410.4557054844662
|
|
},
|
|
{
|
|
"e2e_ms": 16860.906258226023,
|
|
"request_id": 49,
|
|
"tpot_ms": 130.28066064407446,
|
|
"ttft_ms": 315.26235642856903
|
|
},
|
|
{
|
|
"e2e_ms": 16678.317507592936,
|
|
"request_id": 50,
|
|
"tpot_ms": 128.73402984160936,
|
|
"ttft_ms": 329.0957177085474
|
|
},
|
|
{
|
|
"e2e_ms": 16495.414294222966,
|
|
"request_id": 51,
|
|
"tpot_ms": 127.18492295459406,
|
|
"ttft_ms": 342.9290789895205
|
|
},
|
|
{
|
|
"e2e_ms": 16325.716031493954,
|
|
"request_id": 52,
|
|
"tpot_ms": 125.73979205687765,
|
|
"ttft_ms": 356.76244027049364
|
|
},
|
|
{
|
|
"e2e_ms": 16142.044362006885,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.18463433430247,
|
|
"ttft_ms": 370.595801550472
|
|
},
|
|
{
|
|
"e2e_ms": 15942.642543961227,
|
|
"request_id": 54,
|
|
"tpot_ms": 122.50561717425025,
|
|
"ttft_ms": 384.4291628314451
|
|
},
|
|
{
|
|
"e2e_ms": 15727.503328455541,
|
|
"request_id": 55,
|
|
"tpot_ms": 120.70436246394655,
|
|
"ttft_ms": 398.049295534328
|
|
},
|
|
{
|
|
"e2e_ms": 15403.205770995846,
|
|
"request_id": 56,
|
|
"tpot_ms": 118.90039231904261,
|
|
"ttft_ms": 302.8559464774361
|
|
},
|
|
{
|
|
"e2e_ms": 15178.030676088234,
|
|
"request_id": 57,
|
|
"tpot_ms": 117.01843597110097,
|
|
"ttft_ms": 316.6893077584092
|
|
},
|
|
{
|
|
"e2e_ms": 14941.341931765628,
|
|
"request_id": 58,
|
|
"tpot_ms": 115.04582096634839,
|
|
"ttft_ms": 330.5226690393823
|
|
},
|
|
{
|
|
"e2e_ms": 14696.772511761146,
|
|
"request_id": 59,
|
|
"tpot_ms": 113.01115339717941,
|
|
"ttft_ms": 344.3560303193607
|
|
},
|
|
{
|
|
"e2e_ms": 14451.34286402164,
|
|
"request_id": 60,
|
|
"tpot_ms": 110.96971238127013,
|
|
"ttft_ms": 358.1893916003338
|
|
},
|
|
{
|
|
"e2e_ms": 14196.293999590453,
|
|
"request_id": 61,
|
|
"tpot_ms": 108.85252950164681,
|
|
"ttft_ms": 372.02275288130693
|
|
},
|
|
{
|
|
"e2e_ms": 13938.386603616791,
|
|
"request_id": 62,
|
|
"tpot_ms": 106.71283849964966,
|
|
"ttft_ms": 385.8561141612853
|
|
},
|
|
{
|
|
"e2e_ms": 13676.049735660896,
|
|
"request_id": 63,
|
|
"tpot_ms": 104.53992523504405,
|
|
"ttft_ms": 399.4792308103001
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.125,
|
|
"passed": 8
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "3e07b28ac20428d11b8d83064d68561f03804f8adf37111de43398fdd8d94138"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 163.37065601348877,
|
|
"offered_request_rate": 1.6,
|
|
"request_metrics_sha256": "e85085ea5517c1afa9f850cb838a638360d45db695cde8d3c9fef6baf6af44c4",
|
|
"request_rate_per_gpu": 0.2,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 19447.058061600903,
|
|
"request_id": 0,
|
|
"tpot_ms": 150.8298532036487,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 19750.640261300512,
|
|
"request_id": 1,
|
|
"tpot_ms": 152.83450342712686,
|
|
"ttft_ms": 340.65832605540146
|
|
},
|
|
{
|
|
"e2e_ms": 19750.02534103151,
|
|
"request_id": 2,
|
|
"tpot_ms": 153.17602578595117,
|
|
"ttft_ms": 296.67006621570977
|
|
},
|
|
{
|
|
"e2e_ms": 19855.93358266601,
|
|
"request_id": 3,
|
|
"tpot_ms": 153.5864661294274,
|
|
"ttft_ms": 350.4523842287295
|
|
},
|
|
{
|
|
"e2e_ms": 19855.232098385564,
|
|
"request_id": 4,
|
|
"tpot_ms": 153.87573615619343,
|
|
"ttft_ms": 313.01360654900145
|
|
},
|
|
{
|
|
"e2e_ms": 19961.05377600862,
|
|
"request_id": 5,
|
|
"tpot_ms": 154.2244676371592,
|
|
"ttft_ms": 374.54638608940184
|
|
},
|
|
{
|
|
"e2e_ms": 19960.265727716724,
|
|
"request_id": 6,
|
|
"tpot_ms": 154.4718650613629,
|
|
"ttft_ms": 342.33886492363473
|
|
},
|
|
{
|
|
"e2e_ms": 19959.47767942483,
|
|
"request_id": 7,
|
|
"tpot_ms": 154.6679176552935,
|
|
"ttft_ms": 316.6521372025519
|
|
},
|
|
{
|
|
"e2e_ms": 20065.255011109733,
|
|
"request_id": 8,
|
|
"tpot_ms": 154.88778812601134,
|
|
"ttft_ms": 394.50591910629294
|
|
},
|
|
{
|
|
"e2e_ms": 20064.42261687969,
|
|
"request_id": 9,
|
|
"tpot_ms": 155.03615010045385,
|
|
"ttft_ms": 374.8315541220517
|
|
},
|
|
{
|
|
"e2e_ms": 20063.590222649644,
|
|
"request_id": 10,
|
|
"tpot_ms": 155.117974090142,
|
|
"ttft_ms": 363.6075132016101
|
|
},
|
|
{
|
|
"e2e_ms": 20062.7578284196,
|
|
"request_id": 11,
|
|
"tpot_ms": 155.13454318347627,
|
|
"ttft_ms": 360.67084411811567
|
|
},
|
|
{
|
|
"e2e_ms": 20061.925434189554,
|
|
"request_id": 12,
|
|
"tpot_ms": 155.09034115377113,
|
|
"ttft_ms": 365.45210766062206
|
|
},
|
|
{
|
|
"e2e_ms": 20061.09303995951,
|
|
"request_id": 13,
|
|
"tpot_ms": 155.09665427482798,
|
|
"ttft_ms": 363.8179470563543
|
|
},
|
|
{
|
|
"e2e_ms": 20060.260645729464,
|
|
"request_id": 14,
|
|
"tpot_ms": 155.10154332898202,
|
|
"ttft_ms": 362.3646429487479
|
|
},
|
|
{
|
|
"e2e_ms": 20059.42825149942,
|
|
"request_id": 15,
|
|
"tpot_ms": 155.10604675613317,
|
|
"ttft_ms": 360.96031347050683
|
|
},
|
|
{
|
|
"e2e_ms": 20058.595857269374,
|
|
"request_id": 16,
|
|
"tpot_ms": 155.10168661355542,
|
|
"ttft_ms": 360.6816573478344
|
|
},
|
|
{
|
|
"e2e_ms": 20057.76346303933,
|
|
"request_id": 17,
|
|
"tpot_ms": 155.0900405289204,
|
|
"ttft_ms": 361.3283158664373
|
|
},
|
|
{
|
|
"e2e_ms": 20056.931068809285,
|
|
"request_id": 18,
|
|
"tpot_ms": 155.07436184636043,
|
|
"ttft_ms": 362.4871143215085
|
|
},
|
|
{
|
|
"e2e_ms": 20056.09867457924,
|
|
"request_id": 19,
|
|
"tpot_ms": 155.0434256316313,
|
|
"ttft_ms": 365.58361936206654
|
|
},
|
|
{
|
|
"e2e_ms": 20055.26628034948,
|
|
"request_id": 20,
|
|
"tpot_ms": 154.9995151863024,
|
|
"ttft_ms": 370.32785168907446
|
|
},
|
|
{
|
|
"e2e_ms": 20054.433886119732,
|
|
"request_id": 21,
|
|
"tpot_ms": 154.96275487551375,
|
|
"ttft_ms": 374.16401692948574
|
|
},
|
|
{
|
|
"e2e_ms": 20053.601491889985,
|
|
"request_id": 22,
|
|
"tpot_ms": 154.92303029896564,
|
|
"ttft_ms": 378.3766439213494
|
|
},
|
|
{
|
|
"e2e_ms": 20052.76909766024,
|
|
"request_id": 23,
|
|
"tpot_ms": 154.88711594543284,
|
|
"ttft_ms": 382.105372590269
|
|
},
|
|
{
|
|
"e2e_ms": 20051.936703430492,
|
|
"request_id": 24,
|
|
"tpot_ms": 154.8482609975464,
|
|
"ttft_ms": 386.20755674210193
|
|
},
|
|
{
|
|
"e2e_ms": 20051.104309200746,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.85607608193587,
|
|
"ttft_ms": 384.3826467948901
|
|
},
|
|
{
|
|
"e2e_ms": 20050.271914971,
|
|
"request_id": 26,
|
|
"tpot_ms": 154.86595665257255,
|
|
"ttft_ms": 382.2954200942838
|
|
},
|
|
{
|
|
"e2e_ms": 20049.439520741253,
|
|
"request_id": 27,
|
|
"tpot_ms": 154.87441181850545,
|
|
"ttft_ms": 380.3892197910592
|
|
},
|
|
{
|
|
"e2e_ms": 20048.607126511506,
|
|
"request_id": 28,
|
|
"tpot_ms": 154.87585298283847,
|
|
"ttft_ms": 379.37379769102006
|
|
},
|
|
{
|
|
"e2e_ms": 20047.77473228176,
|
|
"request_id": 29,
|
|
"tpot_ms": 154.8790680483812,
|
|
"ttft_ms": 378.13309013734295
|
|
},
|
|
{
|
|
"e2e_ms": 20046.942338052013,
|
|
"request_id": 30,
|
|
"tpot_ms": 154.879633497606,
|
|
"ttft_ms": 377.228883856052
|
|
},
|
|
{
|
|
"e2e_ms": 20046.109943822266,
|
|
"request_id": 31,
|
|
"tpot_ms": 154.8807461594705,
|
|
"ttft_ms": 376.2551815695119
|
|
},
|
|
{
|
|
"e2e_ms": 20045.27754959252,
|
|
"request_id": 32,
|
|
"tpot_ms": 154.8790337660788,
|
|
"ttft_ms": 375.64026130051076
|
|
},
|
|
{
|
|
"e2e_ms": 19846.84310747305,
|
|
"request_id": 33,
|
|
"tpot_ms": 153.32139973576017,
|
|
"ttft_ms": 375.02534103150964
|
|
},
|
|
{
|
|
"e2e_ms": 19648.543544766126,
|
|
"request_id": 34,
|
|
"tpot_ms": 151.76482774805996,
|
|
"ttft_ms": 374.4104207625085
|
|
},
|
|
{
|
|
"e2e_ms": 19450.677394587543,
|
|
"request_id": 35,
|
|
"tpot_ms": 150.2123500638227,
|
|
"ttft_ms": 373.7089364820605
|
|
},
|
|
{
|
|
"e2e_ms": 19253.237256186894,
|
|
"request_id": 36,
|
|
"tpot_ms": 148.66322680303372,
|
|
"ttft_ms": 373.00745220161247
|
|
},
|
|
{
|
|
"e2e_ms": 19054.80482620875,
|
|
"request_id": 37,
|
|
"tpot_ms": 147.10697182912625,
|
|
"ttft_ms": 372.2194039097175
|
|
},
|
|
{
|
|
"e2e_ms": 18856.422732154002,
|
|
"request_id": 38,
|
|
"tpot_ms": 145.55111320107227,
|
|
"ttft_ms": 371.4313556178226
|
|
},
|
|
{
|
|
"e2e_ms": 18658.616312962055,
|
|
"request_id": 39,
|
|
"tpot_ms": 143.99978744595376,
|
|
"ttft_ms": 370.64330732592765
|
|
},
|
|
{
|
|
"e2e_ms": 18469.244245876795,
|
|
"request_id": 40,
|
|
"tpot_ms": 142.51522309276308,
|
|
"ttft_ms": 369.81091309588265
|
|
},
|
|
{
|
|
"e2e_ms": 18279.907702313452,
|
|
"request_id": 41,
|
|
"tpot_ms": 141.0309384523434,
|
|
"ttft_ms": 368.97851886583766
|
|
},
|
|
{
|
|
"e2e_ms": 18091.61193748763,
|
|
"request_id": 42,
|
|
"tpot_ms": 139.55484892009318,
|
|
"ttft_ms": 368.14612463579266
|
|
},
|
|
{
|
|
"e2e_ms": 17903.060652176137,
|
|
"request_id": 43,
|
|
"tpot_ms": 138.07674741551486,
|
|
"ttft_ms": 367.31373040574766
|
|
},
|
|
{
|
|
"e2e_ms": 17716.73280728946,
|
|
"request_id": 44,
|
|
"tpot_ms": 136.61615331585634,
|
|
"ttft_ms": 366.48133617570267
|
|
},
|
|
{
|
|
"e2e_ms": 17529.12437807872,
|
|
"request_id": 45,
|
|
"tpot_ms": 135.14547587506345,
|
|
"ttft_ms": 365.64894194565767
|
|
},
|
|
{
|
|
"e2e_ms": 17339.472913240785,
|
|
"request_id": 46,
|
|
"tpot_ms": 133.65871153956826,
|
|
"ttft_ms": 364.81654771561267
|
|
},
|
|
{
|
|
"e2e_ms": 17149.611868516175,
|
|
"request_id": 47,
|
|
"tpot_ms": 132.17029696874496,
|
|
"ttft_ms": 363.9841534855677
|
|
},
|
|
{
|
|
"e2e_ms": 16958.75366988868,
|
|
"request_id": 48,
|
|
"tpot_ms": 130.67403079238707,
|
|
"ttft_ms": 363.1517592555227
|
|
},
|
|
{
|
|
"e2e_ms": 16767.44725356325,
|
|
"request_id": 49,
|
|
"tpot_ms": 129.1742353428171,
|
|
"ttft_ms": 362.3193650254777
|
|
},
|
|
{
|
|
"e2e_ms": 16576.276975363853,
|
|
"request_id": 50,
|
|
"tpot_ms": 127.67551184699543,
|
|
"ttft_ms": 361.4869707954327
|
|
},
|
|
{
|
|
"e2e_ms": 16384.89705533921,
|
|
"request_id": 51,
|
|
"tpot_ms": 126.1751376281405,
|
|
"ttft_ms": 360.65457656536637
|
|
},
|
|
{
|
|
"e2e_ms": 16202.320435741867,
|
|
"request_id": 52,
|
|
"tpot_ms": 124.74408073548226,
|
|
"ttft_ms": 359.8221823356198
|
|
},
|
|
{
|
|
"e2e_ms": 16010.428211639819,
|
|
"request_id": 53,
|
|
"tpot_ms": 123.23967262625155,
|
|
"ttft_ms": 358.98978810587323
|
|
},
|
|
{
|
|
"e2e_ms": 15808.049221831383,
|
|
"request_id": 54,
|
|
"tpot_ms": 121.6526915587028,
|
|
"ttft_ms": 358.15739387612666
|
|
},
|
|
{
|
|
"e2e_ms": 15595.17863371626,
|
|
"request_id": 55,
|
|
"tpot_ms": 119.98309948086519,
|
|
"ttft_ms": 357.3249996463801
|
|
},
|
|
{
|
|
"e2e_ms": 15377.4072544157,
|
|
"request_id": 56,
|
|
"tpot_ms": 118.27491849605565,
|
|
"ttft_ms": 356.4926054166335
|
|
},
|
|
{
|
|
"e2e_ms": 15157.846080032627,
|
|
"request_id": 57,
|
|
"tpot_ms": 116.55264463658062,
|
|
"ttft_ms": 355.66021118688695
|
|
},
|
|
{
|
|
"e2e_ms": 14930.609139372891,
|
|
"request_id": 58,
|
|
"tpot_ms": 114.76993167256497,
|
|
"ttft_ms": 354.8278169571404
|
|
},
|
|
{
|
|
"e2e_ms": 14698.1184149259,
|
|
"request_id": 59,
|
|
"tpot_ms": 112.9458503322717,
|
|
"ttft_ms": 353.9954227273938
|
|
},
|
|
{
|
|
"e2e_ms": 14465.054205320868,
|
|
"request_id": 60,
|
|
"tpot_ms": 111.11725336081277,
|
|
"ttft_ms": 353.16302849764725
|
|
},
|
|
{
|
|
"e2e_ms": 14225.577184589198,
|
|
"request_id": 61,
|
|
"tpot_ms": 109.23816181355352,
|
|
"ttft_ms": 352.3306342679007
|
|
},
|
|
{
|
|
"e2e_ms": 13984.194476163324,
|
|
"request_id": 62,
|
|
"tpot_ms": 107.34406485137929,
|
|
"ttft_ms": 351.4982400381541
|
|
},
|
|
{
|
|
"e2e_ms": 13739.858786414643,
|
|
"request_id": 63,
|
|
"tpot_ms": 105.42671606776564,
|
|
"ttft_ms": 350.66584580840754
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.140625,
|
|
"passed": 9
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.4375,
|
|
"passed": 28
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "850087527f8ec5b333ae4bb00c5837c85f7dafca77de3d8c80820e071da8dfbb"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 392.34758615493774,
|
|
"offered_request_rate": 2.4,
|
|
"request_metrics_sha256": "c4ae9e9bb2050189ca506883c0c12efd5faecce0ec6249ad641d32f3b9a7f130",
|
|
"request_rate_per_gpu": 0.3,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 25856.75271176486,
|
|
"request_id": 0,
|
|
"tpot_ms": 201.29988981911293,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 26179.284600266838,
|
|
"request_id": 1,
|
|
"tpot_ms": 203.30897363251975,
|
|
"ttft_ms": 359.0449489368311
|
|
},
|
|
{
|
|
"e2e_ms": 25981.87281483019,
|
|
"request_id": 2,
|
|
"tpot_ms": 201.96786246415923,
|
|
"ttft_ms": 331.9542818819672
|
|
},
|
|
{
|
|
"e2e_ms": 25784.35394549511,
|
|
"request_id": 3,
|
|
"tpot_ms": 200.62102535110301,
|
|
"ttft_ms": 305.48372590502584
|
|
},
|
|
{
|
|
"e2e_ms": 25696.083985341404,
|
|
"request_id": 4,
|
|
"tpot_ms": 199.35042104178692,
|
|
"ttft_ms": 378.5805130344673
|
|
},
|
|
{
|
|
"e2e_ms": 25498.25455731707,
|
|
"request_id": 5,
|
|
"tpot_ms": 197.981349219281,
|
|
"ttft_ms": 354.62320646838384
|
|
},
|
|
{
|
|
"e2e_ms": 25300.295013812065,
|
|
"request_id": 6,
|
|
"tpot_ms": 196.59886557857163,
|
|
"ttft_ms": 332.2390853334691
|
|
},
|
|
{
|
|
"e2e_ms": 25102.100957670827,
|
|
"request_id": 7,
|
|
"tpot_ms": 195.19408059762435,
|
|
"ttft_ms": 312.45272177253236
|
|
},
|
|
{
|
|
"e2e_ms": 25012.968951433395,
|
|
"request_id": 8,
|
|
"tpot_ms": 193.83981874911316,
|
|
"ttft_ms": 395.3119702960262
|
|
},
|
|
{
|
|
"e2e_ms": 24814.29543067174,
|
|
"request_id": 9,
|
|
"tpot_ms": 192.41025790813484,
|
|
"ttft_ms": 378.1926763386156
|
|
},
|
|
{
|
|
"e2e_ms": 24615.47335754875,
|
|
"request_id": 10,
|
|
"tpot_ms": 190.95429488392014,
|
|
"ttft_ms": 364.2779072908935
|
|
},
|
|
{
|
|
"e2e_ms": 24416.46039907787,
|
|
"request_id": 11,
|
|
"tpot_ms": 189.47286981272285,
|
|
"ttft_ms": 353.405932862068
|
|
},
|
|
{
|
|
"e2e_ms": 24216.993764693514,
|
|
"request_id": 12,
|
|
"tpot_ms": 187.96377697549212,
|
|
"ttft_ms": 345.59408880601427
|
|
},
|
|
{
|
|
"e2e_ms": 24017.241128749425,
|
|
"request_id": 13,
|
|
"tpot_ms": 186.4682030159872,
|
|
"ttft_ms": 335.7793457190477
|
|
},
|
|
{
|
|
"e2e_ms": 23816.999491057053,
|
|
"request_id": 14,
|
|
"tpot_ms": 184.96817994362138,
|
|
"ttft_ms": 326.0406382171395
|
|
},
|
|
{
|
|
"e2e_ms": 23616.430479744246,
|
|
"request_id": 15,
|
|
"tpot_ms": 183.46465752234423,
|
|
"ttft_ms": 316.41897440652934
|
|
},
|
|
{
|
|
"e2e_ms": 23412.66959473249,
|
|
"request_id": 16,
|
|
"tpot_ms": 181.92890330417333,
|
|
"ttft_ms": 307.69887510247963
|
|
},
|
|
{
|
|
"e2e_ms": 23315.084360363784,
|
|
"request_id": 17,
|
|
"tpot_ms": 180.37041078851854,
|
|
"ttft_ms": 408.042190221928
|
|
},
|
|
{
|
|
"e2e_ms": 23111.095991700124,
|
|
"request_id": 18,
|
|
"tpot_ms": 178.82629774094855,
|
|
"ttft_ms": 400.1561785996568
|
|
},
|
|
{
|
|
"e2e_ms": 22907.001651879975,
|
|
"request_id": 19,
|
|
"tpot_ms": 177.27413055190453,
|
|
"ttft_ms": 393.18707178810007
|
|
},
|
|
{
|
|
"e2e_ms": 22702.9918523004,
|
|
"request_id": 20,
|
|
"tpot_ms": 175.71469647400258,
|
|
"ttft_ms": 387.22540010207496
|
|
},
|
|
{
|
|
"e2e_ms": 22499.024594644914,
|
|
"request_id": 21,
|
|
"tpot_ms": 174.1539791842547,
|
|
"ttft_ms": 381.46923824456815
|
|
},
|
|
{
|
|
"e2e_ms": 22294.98921706886,
|
|
"request_id": 22,
|
|
"tpot_ms": 172.59076723758398,
|
|
"ttft_ms": 375.96177789569475
|
|
},
|
|
{
|
|
"e2e_ms": 22091.055170736552,
|
|
"request_id": 23,
|
|
"tpot_ms": 171.02806584364455,
|
|
"ttft_ms": 370.49080859369354
|
|
},
|
|
{
|
|
"e2e_ms": 21886.829371806663,
|
|
"request_id": 24,
|
|
"tpot_ms": 169.4599867344259,
|
|
"ttft_ms": 365.4110565345743
|
|
},
|
|
{
|
|
"e2e_ms": 21682.517380669786,
|
|
"request_id": 25,
|
|
"tpot_ms": 167.90637996068475,
|
|
"ttft_ms": 358.40712566282207
|
|
},
|
|
{
|
|
"e2e_ms": 21478.40806232245,
|
|
"request_id": 26,
|
|
"tpot_ms": 166.35416808544093,
|
|
"ttft_ms": 351.4287154714495
|
|
},
|
|
{
|
|
"e2e_ms": 21274.47246393178,
|
|
"request_id": 27,
|
|
"tpot_ms": 164.80170050625136,
|
|
"ttft_ms": 344.6564996378552
|
|
},
|
|
{
|
|
"e2e_ms": 21070.655727841604,
|
|
"request_id": 28,
|
|
"tpot_ms": 163.24706150884583,
|
|
"ttft_ms": 338.278916218183
|
|
},
|
|
{
|
|
"e2e_ms": 20866.83848786135,
|
|
"request_id": 29,
|
|
"tpot_ms": 161.69251523328387,
|
|
"ttft_ms": 331.88905323430083
|
|
},
|
|
{
|
|
"e2e_ms": 20663.178771213603,
|
|
"request_id": 30,
|
|
"tpot_ms": 160.1348533340772,
|
|
"ttft_ms": 326.0523977857979
|
|
},
|
|
{
|
|
"e2e_ms": 20459.48769489552,
|
|
"request_id": 31,
|
|
"tpot_ms": 158.57430185728822,
|
|
"ttft_ms": 320.55135901991605
|
|
},
|
|
{
|
|
"e2e_ms": 20255.951788183756,
|
|
"request_id": 32,
|
|
"tpot_ms": 157.01541837295076,
|
|
"ttft_ms": 314.9936548190073
|
|
},
|
|
{
|
|
"e2e_ms": 20052.589009493837,
|
|
"request_id": 33,
|
|
"tpot_ms": 155.45687267778135,
|
|
"ttft_ms": 309.5661794156026
|
|
},
|
|
{
|
|
"e2e_ms": 19955.947670796646,
|
|
"request_id": 34,
|
|
"tpot_ms": 153.8983040696114,
|
|
"ttft_ms": 410.8630539559979
|
|
},
|
|
{
|
|
"e2e_ms": 19752.847929041352,
|
|
"request_id": 35,
|
|
"tpot_ms": 152.3343481355919,
|
|
"ttft_ms": 406.3857158211803
|
|
},
|
|
{
|
|
"e2e_ms": 19549.96119317403,
|
|
"request_id": 36,
|
|
"tpot_ms": 150.77184485827368,
|
|
"ttft_ms": 401.93689617327345
|
|
},
|
|
{
|
|
"e2e_ms": 19346.57831151796,
|
|
"request_id": 37,
|
|
"tpot_ms": 149.20507336304368,
|
|
"ttft_ms": 397.53399441140937
|
|
},
|
|
{
|
|
"e2e_ms": 19143.22059782458,
|
|
"request_id": 38,
|
|
"tpot_ms": 147.63442381272193,
|
|
"ttft_ms": 393.6487736088932
|
|
},
|
|
{
|
|
"e2e_ms": 18940.150721561607,
|
|
"request_id": 39,
|
|
"tpot_ms": 146.06310547094216,
|
|
"ttft_ms": 390.13632675195356
|
|
},
|
|
{
|
|
"e2e_ms": 18741.29802135198,
|
|
"request_id": 40,
|
|
"tpot_ms": 144.5231784918296,
|
|
"ttft_ms": 386.85435288961753
|
|
},
|
|
{
|
|
"e2e_ms": 18542.463082904305,
|
|
"request_id": 41,
|
|
"tpot_ms": 142.98167897793616,
|
|
"ttft_ms": 383.7898527064141
|
|
},
|
|
{
|
|
"e2e_ms": 18344.148533824395,
|
|
"request_id": 42,
|
|
"tpot_ms": 141.4401875845621,
|
|
"ttft_ms": 381.2447105850083
|
|
},
|
|
{
|
|
"e2e_ms": 18145.706224501653,
|
|
"request_id": 43,
|
|
"tpot_ms": 139.89417135289136,
|
|
"ttft_ms": 379.1464626844494
|
|
},
|
|
{
|
|
"e2e_ms": 17948.37563539231,
|
|
"request_id": 44,
|
|
"tpot_ms": 138.3533077037345,
|
|
"ttft_ms": 377.5055570180292
|
|
},
|
|
{
|
|
"e2e_ms": 17750.40475411994,
|
|
"request_id": 45,
|
|
"tpot_ms": 136.80479670989413,
|
|
"ttft_ms": 376.19557196338246
|
|
},
|
|
{
|
|
"e2e_ms": 17551.412355033975,
|
|
"request_id": 46,
|
|
"tpot_ms": 135.2471347218224,
|
|
"ttft_ms": 375.0262453625339
|
|
},
|
|
{
|
|
"e2e_ms": 17352.31516600567,
|
|
"request_id": 47,
|
|
"tpot_ms": 133.6870504253856,
|
|
"ttft_ms": 374.0597619816981
|
|
},
|
|
{
|
|
"e2e_ms": 17152.719400024922,
|
|
"request_id": 48,
|
|
"tpot_ms": 132.11931086353468,
|
|
"ttft_ms": 373.56692035601566
|
|
},
|
|
{
|
|
"e2e_ms": 16952.899525195207,
|
|
"request_id": 49,
|
|
"tpot_ms": 130.53228068835782,
|
|
"ttft_ms": 375.29987777376664
|
|
},
|
|
{
|
|
"e2e_ms": 16753.147719429508,
|
|
"request_id": 50,
|
|
"tpot_ms": 128.94354832701373,
|
|
"ttft_ms": 377.31708189876304
|
|
},
|
|
{
|
|
"e2e_ms": 16553.291092750187,
|
|
"request_id": 51,
|
|
"tpot_ms": 127.34853489391125,
|
|
"ttft_ms": 380.0271612234596
|
|
},
|
|
{
|
|
"e2e_ms": 16357.836116284518,
|
|
"request_id": 52,
|
|
"tpot_ms": 125.77762361641257,
|
|
"ttft_ms": 384.07791700011984
|
|
},
|
|
{
|
|
"e2e_ms": 16157.72333756749,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.16553823707912,
|
|
"ttft_ms": 388.6999814584407
|
|
},
|
|
{
|
|
"e2e_ms": 15952.367175996273,
|
|
"request_id": 54,
|
|
"tpot_ms": 122.51027887411597,
|
|
"ttft_ms": 393.5617589835445
|
|
},
|
|
{
|
|
"e2e_ms": 15741.765215271713,
|
|
"request_id": 55,
|
|
"tpot_ms": 120.81149800475544,
|
|
"ttft_ms": 398.70496866777216
|
|
},
|
|
{
|
|
"e2e_ms": 15528.71285895543,
|
|
"request_id": 56,
|
|
"tpot_ms": 119.08582914503211,
|
|
"ttft_ms": 404.8125575363528
|
|
},
|
|
{
|
|
"e2e_ms": 15314.765605096894,
|
|
"request_id": 57,
|
|
"tpot_ms": 117.34894161957355,
|
|
"ttft_ms": 411.4500194110526
|
|
},
|
|
{
|
|
"e2e_ms": 15096.980468100028,
|
|
"request_id": 58,
|
|
"tpot_ms": 115.58065073324867,
|
|
"ttft_ms": 418.2378249774459
|
|
},
|
|
{
|
|
"e2e_ms": 14778.441120322277,
|
|
"request_id": 59,
|
|
"tpot_ms": 113.87954551567164,
|
|
"ttft_ms": 315.7388398319796
|
|
},
|
|
{
|
|
"e2e_ms": 14557.742348852762,
|
|
"request_id": 60,
|
|
"tpot_ms": 112.08473611057838,
|
|
"ttft_ms": 322.9808628093096
|
|
},
|
|
{
|
|
"e2e_ms": 14333.83717181993,
|
|
"request_id": 61,
|
|
"tpot_ms": 110.26282336485625,
|
|
"ttft_ms": 330.4586044831872
|
|
},
|
|
{
|
|
"e2e_ms": 14108.97915094099,
|
|
"request_id": 62,
|
|
"tpot_ms": 108.43298632152924,
|
|
"ttft_ms": 337.989888106776
|
|
},
|
|
{
|
|
"e2e_ms": 13882.64463939965,
|
|
"request_id": 63,
|
|
"tpot_ms": 106.58813679106942,
|
|
"ttft_ms": 345.9512669338345
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.125,
|
|
"passed": 8
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.421875,
|
|
"passed": 27
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.71875,
|
|
"passed": 46
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "0b9f30d883de83b4af69dd114bd5d2bb974b7fd1f01d45f913bf5049e4b72a86"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 402.0613024234772,
|
|
"offered_request_rate": 3.2,
|
|
"request_metrics_sha256": "48ff1d03e0aeb45bb4395dca86e089cdbe1c7de16b8557fbe8f25e8e65c77def",
|
|
"request_rate_per_gpu": 0.4,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 26482.398681960134,
|
|
"request_id": 0,
|
|
"tpot_ms": 206.22623604112295,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 26389.27725660841,
|
|
"request_id": 1,
|
|
"tpot_ms": 204.89007083645959,
|
|
"ttft_ms": 368.23826037804076
|
|
},
|
|
{
|
|
"e2e_ms": 26186.398050214455,
|
|
"request_id": 2,
|
|
"tpot_ms": 203.43938315354237,
|
|
"ttft_ms": 349.5963897145746
|
|
},
|
|
{
|
|
"e2e_ms": 25983.46530187178,
|
|
"request_id": 3,
|
|
"tpot_ms": 201.98715121187504,
|
|
"ttft_ms": 331.0970979636474
|
|
},
|
|
{
|
|
"e2e_ms": 25780.42643937068,
|
|
"request_id": 4,
|
|
"tpot_ms": 200.53276712329495,
|
|
"ttft_ms": 312.76501471222184
|
|
},
|
|
{
|
|
"e2e_ms": 25577.34505869151,
|
|
"request_id": 5,
|
|
"tpot_ms": 199.0733214464005,
|
|
"ttft_ms": 295.0332349986446
|
|
},
|
|
{
|
|
"e2e_ms": 25483.55218185351,
|
|
"request_id": 6,
|
|
"tpot_ms": 197.69083780569105,
|
|
"ttft_ms": 376.81578053074594
|
|
},
|
|
{
|
|
"e2e_ms": 25280.28848711639,
|
|
"request_id": 7,
|
|
"tpot_ms": 196.22290463044078,
|
|
"ttft_ms": 359.97959905041466
|
|
},
|
|
{
|
|
"e2e_ms": 25076.966707259246,
|
|
"request_id": 8,
|
|
"tpot_ms": 194.75030755695653,
|
|
"ttft_ms": 343.6776475257637
|
|
},
|
|
{
|
|
"e2e_ms": 24873.46328021192,
|
|
"request_id": 9,
|
|
"tpot_ms": 193.27265662306831,
|
|
"ttft_ms": 327.83588908224147
|
|
},
|
|
{
|
|
"e2e_ms": 24669.88557698392,
|
|
"request_id": 10,
|
|
"tpot_ms": 191.78984111033463,
|
|
"ttft_ms": 312.57575597142215
|
|
},
|
|
{
|
|
"e2e_ms": 24466.212431081483,
|
|
"request_id": 11,
|
|
"tpot_ms": 190.30296300954802,
|
|
"ttft_ms": 297.7361288688827
|
|
},
|
|
{
|
|
"e2e_ms": 24370.912463364126,
|
|
"request_id": 12,
|
|
"tpot_ms": 188.79387017231738,
|
|
"ttft_ms": 394.09095147981964
|
|
},
|
|
{
|
|
"e2e_ms": 24166.86947872558,
|
|
"request_id": 13,
|
|
"tpot_ms": 187.2967209572732,
|
|
"ttft_ms": 380.1859171518851
|
|
},
|
|
{
|
|
"e2e_ms": 23962.581993212396,
|
|
"request_id": 14,
|
|
"tpot_ms": 185.79746051386414,
|
|
"ttft_ms": 366.30450795165007
|
|
},
|
|
{
|
|
"e2e_ms": 23758.13082088949,
|
|
"request_id": 15,
|
|
"tpot_ms": 184.29572160562404,
|
|
"ttft_ms": 352.57417697523687
|
|
},
|
|
{
|
|
"e2e_ms": 23552.083711717114,
|
|
"request_id": 16,
|
|
"tpot_ms": 182.7751996506789,
|
|
"ttft_ms": 339.633356080892
|
|
},
|
|
{
|
|
"e2e_ms": 23345.944189149544,
|
|
"request_id": 17,
|
|
"tpot_ms": 181.25255279330477,
|
|
"ttft_ms": 326.8699843998375
|
|
},
|
|
{
|
|
"e2e_ms": 23139.783338151214,
|
|
"request_id": 18,
|
|
"tpot_ms": 179.72694307410856,
|
|
"ttft_ms": 314.461567739424
|
|
},
|
|
{
|
|
"e2e_ms": 22933.56950157464,
|
|
"request_id": 19,
|
|
"tpot_ms": 178.1977153273412,
|
|
"ttft_ms": 302.45965500230557
|
|
},
|
|
{
|
|
"e2e_ms": 22833.726368661068,
|
|
"request_id": 20,
|
|
"tpot_ms": 176.63828124943853,
|
|
"ttft_ms": 400.6646499823745
|
|
},
|
|
{
|
|
"e2e_ms": 22627.576073166823,
|
|
"request_id": 21,
|
|
"tpot_ms": 175.09813975268048,
|
|
"ttft_ms": 390.11232457640244
|
|
},
|
|
{
|
|
"e2e_ms": 22421.391717712297,
|
|
"request_id": 22,
|
|
"tpot_ms": 173.55627478074584,
|
|
"ttft_ms": 379.7448205575735
|
|
},
|
|
{
|
|
"e2e_ms": 22215.258027879143,
|
|
"request_id": 23,
|
|
"tpot_ms": 172.01247264259936,
|
|
"ttft_ms": 369.67400226902566
|
|
},
|
|
{
|
|
"e2e_ms": 22008.978461747698,
|
|
"request_id": 24,
|
|
"tpot_ms": 170.46437149324944,
|
|
"ttft_ms": 360.0032821050201
|
|
},
|
|
{
|
|
"e2e_ms": 21802.65579951293,
|
|
"request_id": 25,
|
|
"tpot_ms": 168.91532250977505,
|
|
"ttft_ms": 350.4098407714995
|
|
},
|
|
{
|
|
"e2e_ms": 21596.434473671965,
|
|
"request_id": 26,
|
|
"tpot_ms": 167.36573728219415,
|
|
"ttft_ms": 340.98583883331025
|
|
},
|
|
{
|
|
"e2e_ms": 21390.300007810118,
|
|
"request_id": 27,
|
|
"tpot_ms": 165.81511332736412,
|
|
"ttft_ms": 331.7806152348748
|
|
},
|
|
{
|
|
"e2e_ms": 21184.224973098888,
|
|
"request_id": 28,
|
|
"tpot_ms": 164.26380332238384,
|
|
"ttft_ms": 322.72195115614187
|
|
},
|
|
{
|
|
"e2e_ms": 20978.149686441437,
|
|
"request_id": 29,
|
|
"tpot_ms": 162.71174941720716,
|
|
"ttft_ms": 313.7575104561279
|
|
},
|
|
{
|
|
"e2e_ms": 20772.153161450715,
|
|
"request_id": 30,
|
|
"tpot_ms": 161.15510654351164,
|
|
"ttft_ms": 305.4546304247392
|
|
},
|
|
{
|
|
"e2e_ms": 20672.62875180034,
|
|
"request_id": 31,
|
|
"tpot_ms": 159.59455506672822,
|
|
"ttft_ms": 404.12025832585744
|
|
},
|
|
{
|
|
"e2e_ms": 20466.69413177714,
|
|
"request_id": 32,
|
|
"tpot_ms": 158.03533915661183,
|
|
"ttft_ms": 396.2060588874419
|
|
},
|
|
{
|
|
"e2e_ms": 20260.84607576572,
|
|
"request_id": 33,
|
|
"tpot_ms": 156.475168525323,
|
|
"ttft_ms": 388.49967304969545
|
|
},
|
|
{
|
|
"e2e_ms": 20055.02118508905,
|
|
"request_id": 34,
|
|
"tpot_ms": 154.9132321630232,
|
|
"ttft_ms": 381.04070038510326
|
|
},
|
|
{
|
|
"e2e_ms": 19849.304647544788,
|
|
"request_id": 35,
|
|
"tpot_ms": 153.3461190070677,
|
|
"ttft_ms": 374.34753364718887
|
|
},
|
|
{
|
|
"e2e_ms": 19643.694612944317,
|
|
"request_id": 36,
|
|
"tpot_ms": 151.77962188577303,
|
|
"ttft_ms": 367.6826334511443
|
|
},
|
|
{
|
|
"e2e_ms": 19437.836505449406,
|
|
"request_id": 37,
|
|
"tpot_ms": 150.210341914627,
|
|
"ttft_ms": 361.1230822917744
|
|
},
|
|
{
|
|
"e2e_ms": 19231.99098193562,
|
|
"request_id": 38,
|
|
"tpot_ms": 148.63640086508497,
|
|
"ttft_ms": 355.16807206983003
|
|
},
|
|
{
|
|
"e2e_ms": 19026.289377137648,
|
|
"request_id": 39,
|
|
"tpot_ms": 147.05985988148976,
|
|
"ttft_ms": 349.6871721884496
|
|
},
|
|
{
|
|
"e2e_ms": 18822.696360365982,
|
|
"request_id": 40,
|
|
"tpot_ms": 145.49844654462956,
|
|
"ttft_ms": 344.3936491980288
|
|
},
|
|
{
|
|
"e2e_ms": 18619.112224474804,
|
|
"request_id": 41,
|
|
"tpot_ms": 143.93654065290042,
|
|
"ttft_ms": 339.17156155644966
|
|
},
|
|
{
|
|
"e2e_ms": 18415.78828326805,
|
|
"request_id": 42,
|
|
"tpot_ms": 142.37219516274803,
|
|
"ttft_ms": 334.51949759905375
|
|
},
|
|
{
|
|
"e2e_ms": 18212.400461939902,
|
|
"request_id": 43,
|
|
"tpot_ms": 140.8040960160447,
|
|
"ttft_ms": 330.280267902225
|
|
},
|
|
{
|
|
"e2e_ms": 18009.568500717836,
|
|
"request_id": 44,
|
|
"tpot_ms": 139.2366051127279,
|
|
"ttft_ms": 326.5196514013962
|
|
},
|
|
{
|
|
"e2e_ms": 17806.416393414805,
|
|
"request_id": 45,
|
|
"tpot_ms": 137.66365486442547,
|
|
"ttft_ms": 323.1322256327704
|
|
},
|
|
{
|
|
"e2e_ms": 17602.75352720573,
|
|
"request_id": 46,
|
|
"tpot_ms": 136.08599255484802,
|
|
"ttft_ms": 319.83247274003014
|
|
},
|
|
{
|
|
"e2e_ms": 17399.03826602459,
|
|
"request_id": 47,
|
|
"tpot_ms": 134.50648843613666,
|
|
"ttft_ms": 316.71423463523496
|
|
},
|
|
{
|
|
"e2e_ms": 17195.073716367715,
|
|
"request_id": 48,
|
|
"tpot_ms": 132.92201961792392,
|
|
"ttft_ms": 313.9772248913761
|
|
},
|
|
{
|
|
"e2e_ms": 16990.997112286357,
|
|
"request_id": 49,
|
|
"tpot_ms": 131.3320936883647,
|
|
"ttft_ms": 311.8212138640413
|
|
},
|
|
{
|
|
"e2e_ms": 16895.41197318666,
|
|
"request_id": 50,
|
|
"tpot_ms": 129.74336132702078,
|
|
"ttft_ms": 418.0050846550216
|
|
},
|
|
{
|
|
"e2e_ms": 16691.3169931805,
|
|
"request_id": 51,
|
|
"tpot_ms": 128.14881090811454,
|
|
"ttft_ms": 416.418007849952
|
|
},
|
|
{
|
|
"e2e_ms": 16489.422838281163,
|
|
"request_id": 52,
|
|
"tpot_ms": 126.56215930365238,
|
|
"ttft_ms": 416.0286067173118
|
|
},
|
|
{
|
|
"e2e_ms": 16285.199782255653,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.95445752714085,
|
|
"ttft_ms": 415.9836763087661
|
|
},
|
|
{
|
|
"e2e_ms": 16078.355034803544,
|
|
"request_id": 54,
|
|
"tpot_ms": 123.3249765236981,
|
|
"ttft_ms": 416.08301629388575
|
|
},
|
|
{
|
|
"e2e_ms": 15868.887387774763,
|
|
"request_id": 55,
|
|
"tpot_ms": 121.67321161824907,
|
|
"ttft_ms": 416.3895122571297
|
|
},
|
|
{
|
|
"e2e_ms": 15658.194542949623,
|
|
"request_id": 56,
|
|
"tpot_ms": 120.00563624201372,
|
|
"ttft_ms": 417.4787402138804
|
|
},
|
|
{
|
|
"e2e_ms": 15447.054249353854,
|
|
"request_id": 57,
|
|
"tpot_ms": 118.33082151865976,
|
|
"ttft_ms": 419.0399164840635
|
|
},
|
|
{
|
|
"e2e_ms": 15233.99501418892,
|
|
"request_id": 58,
|
|
"tpot_ms": 116.64063648866745,
|
|
"ttft_ms": 420.6341801281539
|
|
},
|
|
{
|
|
"e2e_ms": 15019.622333077174,
|
|
"request_id": 59,
|
|
"tpot_ms": 114.93953127109042,
|
|
"ttft_ms": 422.3018616486911
|
|
},
|
|
{
|
|
"e2e_ms": 14805.106280675915,
|
|
"request_id": 60,
|
|
"tpot_ms": 113.23514577866216,
|
|
"ttft_ms": 424.2427667858202
|
|
},
|
|
{
|
|
"e2e_ms": 14588.987025492997,
|
|
"request_id": 61,
|
|
"tpot_ms": 111.5171161341123,
|
|
"ttft_ms": 426.31327646073555
|
|
},
|
|
{
|
|
"e2e_ms": 14372.391348386529,
|
|
"request_id": 62,
|
|
"tpot_ms": 109.79194855942059,
|
|
"ttft_ms": 428.8138813401155
|
|
},
|
|
{
|
|
"e2e_ms": 14155.057425949359,
|
|
"request_id": 63,
|
|
"tpot_ms": 108.05983074662959,
|
|
"ttft_ms": 431.45892112740114
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.109375,
|
|
"passed": 7
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.40625,
|
|
"passed": 26
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.71875,
|
|
"passed": 46
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "96c4e04e8f133e66449a5567fcde568f27af44bd9f5d1cadeade66b771a8f0f3"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"loads": [
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 774.4871437549591,
|
|
"offered_request_rate": 0.1,
|
|
"request_metrics_sha256": "1146954f1d73891536473385a3ecb02b14b7eaeee231291f2cbe6f9d4d222414",
|
|
"request_rate_per_gpu": 0.0125,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12579.084570749343,
|
|
"request_id": 0,
|
|
"tpot_ms": 96.75132177962064,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 12682.66705927438,
|
|
"request_id": 1,
|
|
"tpot_ms": 96.90664947750408,
|
|
"ttft_ms": 375.5225756313614
|
|
},
|
|
{
|
|
"e2e_ms": 12692.014437909165,
|
|
"request_id": 2,
|
|
"tpot_ms": 96.91832475611541,
|
|
"ttft_ms": 383.3871938825091
|
|
},
|
|
{
|
|
"e2e_ms": 12605.650216029226,
|
|
"request_id": 3,
|
|
"tpot_ms": 96.9198422551915,
|
|
"ttft_ms": 296.8302496199051
|
|
},
|
|
{
|
|
"e2e_ms": 12614.259349375281,
|
|
"request_id": 4,
|
|
"tpot_ms": 96.9125118011697,
|
|
"ttft_ms": 306.3703506267288
|
|
},
|
|
{
|
|
"e2e_ms": 12622.868482721336,
|
|
"request_id": 5,
|
|
"tpot_ms": 96.9125118011697,
|
|
"ttft_ms": 314.97948397278464
|
|
},
|
|
{
|
|
"e2e_ms": 12631.47761606615,
|
|
"request_id": 6,
|
|
"tpot_ms": 96.91251180115991,
|
|
"ttft_ms": 323.5886173188405
|
|
},
|
|
{
|
|
"e2e_ms": 12640.086749411268,
|
|
"request_id": 7,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 332.19775066687873
|
|
},
|
|
{
|
|
"e2e_ms": 12648.695882756385,
|
|
"request_id": 8,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 340.80688401199666
|
|
},
|
|
{
|
|
"e2e_ms": 12657.305016101503,
|
|
"request_id": 9,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 349.4160173571146
|
|
},
|
|
{
|
|
"e2e_ms": 12665.91414944662,
|
|
"request_id": 10,
|
|
"tpot_ms": 96.9125118011369,
|
|
"ttft_ms": 358.0251507022325
|
|
},
|
|
{
|
|
"e2e_ms": 12674.523282791739,
|
|
"request_id": 11,
|
|
"tpot_ms": 96.91251180113692,
|
|
"ttft_ms": 366.63428404735043
|
|
},
|
|
{
|
|
"e2e_ms": 12683.132416062932,
|
|
"request_id": 12,
|
|
"tpot_ms": 96.91251180055482,
|
|
"ttft_ms": 375.24341739246836
|
|
},
|
|
{
|
|
"e2e_ms": 12692.479794575775,
|
|
"request_id": 13,
|
|
"tpot_ms": 96.91832475507798,
|
|
"ttft_ms": 383.85255068087076
|
|
},
|
|
{
|
|
"e2e_ms": 12606.11557253071,
|
|
"request_id": 14,
|
|
"tpot_ms": 96.91984225378269,
|
|
"ttft_ms": 297.2956063003096
|
|
},
|
|
{
|
|
"e2e_ms": 12614.724705710416,
|
|
"request_id": 15,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 306.8357071486503
|
|
},
|
|
{
|
|
"e2e_ms": 12623.33383889012,
|
|
"request_id": 16,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 315.44484032835385
|
|
},
|
|
{
|
|
"e2e_ms": 12631.942972069823,
|
|
"request_id": 17,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 324.0539735080574
|
|
},
|
|
{
|
|
"e2e_ms": 12640.552105249526,
|
|
"request_id": 18,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 332.663106687761
|
|
},
|
|
{
|
|
"e2e_ms": 12649.16123842923,
|
|
"request_id": 19,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 341.2722398674646
|
|
},
|
|
{
|
|
"e2e_ms": 12657.770371608933,
|
|
"request_id": 20,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 349.88137304716815
|
|
},
|
|
{
|
|
"e2e_ms": 12666.379504788636,
|
|
"request_id": 21,
|
|
"tpot_ms": 96.91251179969893,
|
|
"ttft_ms": 358.49050622687173
|
|
},
|
|
{
|
|
"e2e_ms": 12674.988637968341,
|
|
"request_id": 22,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 367.0996394065753
|
|
},
|
|
{
|
|
"e2e_ms": 12683.597771148045,
|
|
"request_id": 23,
|
|
"tpot_ms": 96.91251179969895,
|
|
"ttft_ms": 375.7087725862789
|
|
},
|
|
{
|
|
"e2e_ms": 12692.945149660885,
|
|
"request_id": 24,
|
|
"tpot_ms": 96.91832475507798,
|
|
"ttft_ms": 384.31790576598246
|
|
},
|
|
{
|
|
"e2e_ms": 12606.580927732637,
|
|
"request_id": 25,
|
|
"tpot_ms": 96.91984225470249,
|
|
"ttft_ms": 297.7609613854213
|
|
},
|
|
{
|
|
"e2e_ms": 12615.190061213469,
|
|
"request_id": 26,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 307.3010623465393
|
|
},
|
|
{
|
|
"e2e_ms": 12623.7991946943,
|
|
"request_id": 27,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 315.9101958273709
|
|
},
|
|
{
|
|
"e2e_ms": 12632.408328175132,
|
|
"request_id": 28,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 324.5193293082025
|
|
},
|
|
{
|
|
"e2e_ms": 12641.017461655963,
|
|
"request_id": 29,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 333.1284627890341
|
|
},
|
|
{
|
|
"e2e_ms": 12649.626595136795,
|
|
"request_id": 30,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 341.73759626986566
|
|
},
|
|
{
|
|
"e2e_ms": 12658.235728617627,
|
|
"request_id": 31,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 350.34672975069725
|
|
},
|
|
{
|
|
"e2e_ms": 12666.844862098458,
|
|
"request_id": 32,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 358.95586323152884
|
|
},
|
|
{
|
|
"e2e_ms": 12675.45399557929,
|
|
"request_id": 33,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 367.5649967123604
|
|
},
|
|
{
|
|
"e2e_ms": 12684.063129060121,
|
|
"request_id": 34,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 376.174130193192
|
|
},
|
|
{
|
|
"e2e_ms": 12693.41050787034,
|
|
"request_id": 35,
|
|
"tpot_ms": 96.9183247574513,
|
|
"ttft_ms": 384.7832636740236
|
|
},
|
|
{
|
|
"e2e_ms": 12607.04628612126,
|
|
"request_id": 36,
|
|
"tpot_ms": 96.91984225613498,
|
|
"ttft_ms": 298.22631959211776
|
|
},
|
|
{
|
|
"e2e_ms": 12615.655419602092,
|
|
"request_id": 37,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 307.76642073516314
|
|
},
|
|
{
|
|
"e2e_ms": 12624.264553082923,
|
|
"request_id": 38,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 316.3755542159947
|
|
},
|
|
{
|
|
"e2e_ms": 12632.873686563755,
|
|
"request_id": 39,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 324.9846876968263
|
|
},
|
|
{
|
|
"e2e_ms": 12641.482820044586,
|
|
"request_id": 40,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 333.5938211776579
|
|
},
|
|
{
|
|
"e2e_ms": 12650.091953525418,
|
|
"request_id": 41,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 342.2029546584895
|
|
},
|
|
{
|
|
"e2e_ms": 12658.70108700625,
|
|
"request_id": 42,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 350.8120881393211
|
|
},
|
|
{
|
|
"e2e_ms": 12667.310220487081,
|
|
"request_id": 43,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 359.42122162015266
|
|
},
|
|
{
|
|
"e2e_ms": 12675.919353967913,
|
|
"request_id": 44,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 368.03035510098425
|
|
},
|
|
{
|
|
"e2e_ms": 12684.528487448744,
|
|
"request_id": 45,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 376.63948858181584
|
|
},
|
|
{
|
|
"e2e_ms": 12693.875866258964,
|
|
"request_id": 46,
|
|
"tpot_ms": 96.91832475745132,
|
|
"ttft_ms": 385.2486220626474
|
|
},
|
|
{
|
|
"e2e_ms": 12607.511644509885,
|
|
"request_id": 47,
|
|
"tpot_ms": 96.91984225613498,
|
|
"ttft_ms": 298.6916779807416
|
|
},
|
|
{
|
|
"e2e_ms": 12616.120777990716,
|
|
"request_id": 48,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 308.23177912378696
|
|
},
|
|
{
|
|
"e2e_ms": 12624.729911471548,
|
|
"request_id": 49,
|
|
"tpot_ms": 96.9125118021018,
|
|
"ttft_ms": 316.84091260461855
|
|
},
|
|
{
|
|
"e2e_ms": 12633.339044941862,
|
|
"request_id": 50,
|
|
"tpot_ms": 96.91251180201898,
|
|
"ttft_ms": 325.45004608545014
|
|
},
|
|
{
|
|
"e2e_ms": 12641.94817818975,
|
|
"request_id": 51,
|
|
"tpot_ms": 96.91251180018479,
|
|
"ttft_ms": 334.0591795662817
|
|
},
|
|
{
|
|
"e2e_ms": 12650.557311437638,
|
|
"request_id": 52,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 342.66831284185173
|
|
},
|
|
{
|
|
"e2e_ms": 12659.166444685525,
|
|
"request_id": 53,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 351.277446089739
|
|
},
|
|
{
|
|
"e2e_ms": 12667.775577933411,
|
|
"request_id": 54,
|
|
"tpot_ms": 96.91251179996681,
|
|
"ttft_ms": 359.88657933762624
|
|
},
|
|
{
|
|
"e2e_ms": 12676.3847111813,
|
|
"request_id": 55,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 368.4957125855135
|
|
},
|
|
{
|
|
"e2e_ms": 12684.993844429187,
|
|
"request_id": 56,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 377.10484583340076
|
|
},
|
|
{
|
|
"e2e_ms": 12694.3412230047,
|
|
"request_id": 57,
|
|
"tpot_ms": 96.91832475530246,
|
|
"ttft_ms": 385.713979081288
|
|
},
|
|
{
|
|
"e2e_ms": 12607.977001027848,
|
|
"request_id": 58,
|
|
"tpot_ms": 96.9198422540282,
|
|
"ttft_ms": 299.1570347662673
|
|
},
|
|
{
|
|
"e2e_ms": 12616.586134275734,
|
|
"request_id": 59,
|
|
"tpot_ms": 96.91251179996681,
|
|
"ttft_ms": 308.6971356799495
|
|
},
|
|
{
|
|
"e2e_ms": 12625.195267523623,
|
|
"request_id": 60,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 317.30626892783675
|
|
},
|
|
{
|
|
"e2e_ms": 12633.80440077151,
|
|
"request_id": 61,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 325.915402175724
|
|
},
|
|
{
|
|
"e2e_ms": 12642.413534019397,
|
|
"request_id": 62,
|
|
"tpot_ms": 96.91251179996682,
|
|
"ttft_ms": 334.52453542361127
|
|
},
|
|
{
|
|
"e2e_ms": 12436.81062947212,
|
|
"request_id": 63,
|
|
"tpot_ms": 95.22580284094978,
|
|
"ttft_ms": 343.1336686714985
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "37e166453683700c9df7abffa278431119205a29c83e92211ffa69d90e1dca0e"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 403.8260774612427,
|
|
"offered_request_rate": 0.2,
|
|
"request_metrics_sha256": "cf8a095cdf380d42316d1ad169ce9b2b45ae7d64348cd38271a332663de349ad",
|
|
"request_rate_per_gpu": 0.025,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 12823.625057543633,
|
|
"request_id": 0,
|
|
"tpot_ms": 98.67683742367018,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 12920.533566322107,
|
|
"request_id": 1,
|
|
"tpot_ms": 99.10315669196875,
|
|
"ttft_ms": 334.4326664420745
|
|
},
|
|
{
|
|
"e2e_ms": 12921.724204826443,
|
|
"request_id": 2,
|
|
"tpot_ms": 99.22365133274592,
|
|
"ttft_ms": 320.32048556771286
|
|
},
|
|
{
|
|
"e2e_ms": 12922.91484333078,
|
|
"request_id": 3,
|
|
"tpot_ms": 99.22887081733072,
|
|
"ttft_ms": 320.84824952977976
|
|
},
|
|
{
|
|
"e2e_ms": 12924.105481838353,
|
|
"request_id": 4,
|
|
"tpot_ms": 99.22887081736695,
|
|
"ttft_ms": 322.0388880327505
|
|
},
|
|
{
|
|
"e2e_ms": 12925.296120352506,
|
|
"request_id": 5,
|
|
"tpot_ms": 99.22887081744425,
|
|
"ttft_ms": 323.2295265370873
|
|
},
|
|
{
|
|
"e2e_ms": 12926.486758866658,
|
|
"request_id": 6,
|
|
"tpot_ms": 99.22887081752154,
|
|
"ttft_ms": 324.420165041424
|
|
},
|
|
{
|
|
"e2e_ms": 12927.677397380812,
|
|
"request_id": 7,
|
|
"tpot_ms": 99.22887081749418,
|
|
"ttft_ms": 325.6108035590515
|
|
},
|
|
{
|
|
"e2e_ms": 12928.868035894964,
|
|
"request_id": 8,
|
|
"tpot_ms": 99.22887081749417,
|
|
"ttft_ms": 326.8014420732044
|
|
},
|
|
{
|
|
"e2e_ms": 12930.058674409118,
|
|
"request_id": 9,
|
|
"tpot_ms": 99.22887081749418,
|
|
"ttft_ms": 327.9920805873573
|
|
},
|
|
{
|
|
"e2e_ms": 12931.24931292327,
|
|
"request_id": 10,
|
|
"tpot_ms": 99.22887081749417,
|
|
"ttft_ms": 329.1827191015102
|
|
},
|
|
{
|
|
"e2e_ms": 12932.439951441467,
|
|
"request_id": 11,
|
|
"tpot_ms": 99.22887081752602,
|
|
"ttft_ms": 330.3733576156631
|
|
},
|
|
{
|
|
"e2e_ms": 12933.63058995827,
|
|
"request_id": 12,
|
|
"tpot_ms": 99.22887081754688,
|
|
"ttft_ms": 331.563996129816
|
|
},
|
|
{
|
|
"e2e_ms": 12934.821228475073,
|
|
"request_id": 13,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 332.7546346417307
|
|
},
|
|
{
|
|
"e2e_ms": 12936.011866991876,
|
|
"request_id": 14,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 333.94527315853395
|
|
},
|
|
{
|
|
"e2e_ms": 12937.20250550868,
|
|
"request_id": 15,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 335.1359116753372
|
|
},
|
|
{
|
|
"e2e_ms": 12938.393144025482,
|
|
"request_id": 16,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 336.3265501921404
|
|
},
|
|
{
|
|
"e2e_ms": 12939.583782542286,
|
|
"request_id": 17,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 337.51718870894365
|
|
},
|
|
{
|
|
"e2e_ms": 12940.77442105909,
|
|
"request_id": 18,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 338.7078272257469
|
|
},
|
|
{
|
|
"e2e_ms": 12941.965059575892,
|
|
"request_id": 19,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 339.8984657425501
|
|
},
|
|
{
|
|
"e2e_ms": 12943.155698092696,
|
|
"request_id": 20,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 341.08910425935335
|
|
},
|
|
{
|
|
"e2e_ms": 12944.346336609498,
|
|
"request_id": 21,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 342.2797427761566
|
|
},
|
|
{
|
|
"e2e_ms": 12945.536975126302,
|
|
"request_id": 22,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 343.4703812929598
|
|
},
|
|
{
|
|
"e2e_ms": 12946.727613643105,
|
|
"request_id": 23,
|
|
"tpot_ms": 99.22887081758537,
|
|
"ttft_ms": 344.66101980976305
|
|
},
|
|
{
|
|
"e2e_ms": 12947.918252142103,
|
|
"request_id": 24,
|
|
"tpot_ms": 99.22887081744517,
|
|
"ttft_ms": 345.8516583265663
|
|
},
|
|
{
|
|
"e2e_ms": 12949.10889064036,
|
|
"request_id": 25,
|
|
"tpot_ms": 99.22887081729912,
|
|
"ttft_ms": 347.0422968433695
|
|
},
|
|
{
|
|
"e2e_ms": 12950.299529138618,
|
|
"request_id": 26,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 348.23293534631716
|
|
},
|
|
{
|
|
"e2e_ms": 12951.490167636877,
|
|
"request_id": 27,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 349.42357384457523
|
|
},
|
|
{
|
|
"e2e_ms": 12952.680806135135,
|
|
"request_id": 28,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 350.6142123428333
|
|
},
|
|
{
|
|
"e2e_ms": 12953.871444633392,
|
|
"request_id": 29,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 351.80485084109137
|
|
},
|
|
{
|
|
"e2e_ms": 12955.06208313165,
|
|
"request_id": 30,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 352.99548933934943
|
|
},
|
|
{
|
|
"e2e_ms": 12956.25272162991,
|
|
"request_id": 31,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 354.1861278376075
|
|
},
|
|
{
|
|
"e2e_ms": 12957.443360128167,
|
|
"request_id": 32,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 355.37676633586557
|
|
},
|
|
{
|
|
"e2e_ms": 12958.633998626425,
|
|
"request_id": 33,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 356.56740483412364
|
|
},
|
|
{
|
|
"e2e_ms": 12959.824637124682,
|
|
"request_id": 34,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 357.7580433323817
|
|
},
|
|
{
|
|
"e2e_ms": 12961.015275622942,
|
|
"request_id": 35,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 358.94868183063977
|
|
},
|
|
{
|
|
"e2e_ms": 12962.2059141212,
|
|
"request_id": 36,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 360.13932032889784
|
|
},
|
|
{
|
|
"e2e_ms": 12963.396552619457,
|
|
"request_id": 37,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 361.3299588271559
|
|
},
|
|
{
|
|
"e2e_ms": 12964.587191117715,
|
|
"request_id": 38,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 362.520597325414
|
|
},
|
|
{
|
|
"e2e_ms": 12965.777829615974,
|
|
"request_id": 39,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 363.71123582367204
|
|
},
|
|
{
|
|
"e2e_ms": 12966.968468114232,
|
|
"request_id": 40,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 364.9018743219301
|
|
},
|
|
{
|
|
"e2e_ms": 12968.15910661249,
|
|
"request_id": 41,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 366.0925128201882
|
|
},
|
|
{
|
|
"e2e_ms": 12969.349745110747,
|
|
"request_id": 42,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 367.28315131844624
|
|
},
|
|
{
|
|
"e2e_ms": 12970.540383609006,
|
|
"request_id": 43,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 368.4737898167043
|
|
},
|
|
{
|
|
"e2e_ms": 12971.731022107264,
|
|
"request_id": 44,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 369.6644283149624
|
|
},
|
|
{
|
|
"e2e_ms": 12972.921660605522,
|
|
"request_id": 45,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 370.85506681322045
|
|
},
|
|
{
|
|
"e2e_ms": 12974.11229910378,
|
|
"request_id": 46,
|
|
"tpot_ms": 99.2288708172622,
|
|
"ttft_ms": 372.0457053114785
|
|
},
|
|
{
|
|
"e2e_ms": 12975.302937602039,
|
|
"request_id": 47,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 373.2363438097366
|
|
},
|
|
{
|
|
"e2e_ms": 12976.493576100296,
|
|
"request_id": 48,
|
|
"tpot_ms": 99.22887081726222,
|
|
"ttft_ms": 374.42698230799465
|
|
},
|
|
{
|
|
"e2e_ms": 12977.684214615067,
|
|
"request_id": 49,
|
|
"tpot_ms": 99.22887081739223,
|
|
"ttft_ms": 375.6176208062527
|
|
},
|
|
{
|
|
"e2e_ms": 12978.874853139188,
|
|
"request_id": 50,
|
|
"tpot_ms": 99.22887081759589,
|
|
"ttft_ms": 376.8082593045108
|
|
},
|
|
{
|
|
"e2e_ms": 12980.06549166331,
|
|
"request_id": 51,
|
|
"tpot_ms": 99.22887081779955,
|
|
"ttft_ms": 377.99889780276885
|
|
},
|
|
{
|
|
"e2e_ms": 12981.256130187432,
|
|
"request_id": 52,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 379.189536321519
|
|
},
|
|
{
|
|
"e2e_ms": 12982.446768711554,
|
|
"request_id": 53,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 380.3801748456408
|
|
},
|
|
{
|
|
"e2e_ms": 12983.637407235676,
|
|
"request_id": 54,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 381.5708133697626
|
|
},
|
|
{
|
|
"e2e_ms": 12984.828045759798,
|
|
"request_id": 55,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 382.76145189388444
|
|
},
|
|
{
|
|
"e2e_ms": 12986.01868428392,
|
|
"request_id": 56,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 383.95209041800626
|
|
},
|
|
{
|
|
"e2e_ms": 12987.209322808041,
|
|
"request_id": 57,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 385.1427289421281
|
|
},
|
|
{
|
|
"e2e_ms": 12988.399961332163,
|
|
"request_id": 58,
|
|
"tpot_ms": 99.22887081784184,
|
|
"ttft_ms": 386.3333674662499
|
|
},
|
|
{
|
|
"e2e_ms": 12989.881679406835,
|
|
"request_id": 59,
|
|
"tpot_ms": 99.2311627828068,
|
|
"ttft_ms": 387.52400599037173
|
|
},
|
|
{
|
|
"e2e_ms": 12991.363397481507,
|
|
"request_id": 60,
|
|
"tpot_ms": 99.23345474777176,
|
|
"ttft_ms": 388.71464451449356
|
|
},
|
|
{
|
|
"e2e_ms": 12896.836165730803,
|
|
"request_id": 61,
|
|
"tpot_ms": 99.23492287861664,
|
|
"ttft_ms": 294.0009601464908
|
|
},
|
|
{
|
|
"e2e_ms": 12689.81487892944,
|
|
"request_id": 62,
|
|
"tpot_ms": 97.59316693471084,
|
|
"ttft_ms": 295.4826782211626
|
|
},
|
|
{
|
|
"e2e_ms": 12448.118757066311,
|
|
"request_id": 63,
|
|
"tpot_ms": 95.67691266262345,
|
|
"ttft_ms": 297.1508489131338
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "e6cebfc858981b2ae4632d2c75f5e8c1f83280e7306351e15c7cd611d2664d80"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 222.21414136886597,
|
|
"offered_request_rate": 0.4,
|
|
"request_metrics_sha256": "e09d047b2039786ee02cd4c59c081ca95e5197ae65723b6170e43f6c0924921a",
|
|
"request_rate_per_gpu": 0.05,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 13556.392075191316,
|
|
"request_id": 0,
|
|
"tpot_ms": 104.4466564602661,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 13715.056219396885,
|
|
"request_id": 1,
|
|
"tpot_ms": 105.11756526634255,
|
|
"ttft_ms": 365.12543057138026
|
|
},
|
|
{
|
|
"e2e_ms": 13775.965021971426,
|
|
"request_id": 2,
|
|
"tpot_ms": 105.6336255570719,
|
|
"ttft_ms": 360.4945762232949
|
|
},
|
|
{
|
|
"e2e_ms": 13838.18765138338,
|
|
"request_id": 3,
|
|
"tpot_ms": 106.07242606904647,
|
|
"ttft_ms": 366.98954061447785
|
|
},
|
|
{
|
|
"e2e_ms": 13802.282961907104,
|
|
"request_id": 4,
|
|
"tpot_ms": 106.21712281117094,
|
|
"ttft_ms": 312.70836488839393
|
|
},
|
|
{
|
|
"e2e_ms": 13864.505591319055,
|
|
"request_id": 5,
|
|
"tpot_ms": 106.32495967758555,
|
|
"ttft_ms": 361.2357122656906
|
|
},
|
|
{
|
|
"e2e_ms": 13828.78708091747,
|
|
"request_id": 6,
|
|
"tpot_ms": 106.35892091285005,
|
|
"ttft_ms": 321.20412498551156
|
|
},
|
|
{
|
|
"e2e_ms": 13892.323156276025,
|
|
"request_id": 7,
|
|
"tpot_ms": 106.3810710779423,
|
|
"ttft_ms": 381.9271293773525
|
|
},
|
|
{
|
|
"e2e_ms": 13856.60464587857,
|
|
"request_id": 8,
|
|
"tpot_ms": 106.3914161711828,
|
|
"ttft_ms": 344.89479213835494
|
|
},
|
|
{
|
|
"e2e_ms": 13820.886135481267,
|
|
"request_id": 9,
|
|
"tpot_ms": 106.39288214818258,
|
|
"ttft_ms": 308.9901026620794
|
|
},
|
|
{
|
|
"e2e_ms": 13883.108764897528,
|
|
"request_id": 10,
|
|
"tpot_ms": 106.39288214821651,
|
|
"ttft_ms": 371.21273207403146
|
|
},
|
|
{
|
|
"e2e_ms": 13847.390254500226,
|
|
"request_id": 11,
|
|
"tpot_ms": 106.39288214825024,
|
|
"ttft_ms": 335.4942216724446
|
|
},
|
|
{
|
|
"e2e_ms": 13811.485565028235,
|
|
"request_id": 12,
|
|
"tpot_ms": 106.39288214828399,
|
|
"ttft_ms": 299.589532196169
|
|
},
|
|
{
|
|
"e2e_ms": 13873.708194444496,
|
|
"request_id": 13,
|
|
"tpot_ms": 106.38107407728566,
|
|
"ttft_ms": 363.31178662921815
|
|
},
|
|
{
|
|
"e2e_ms": 13837.989684047194,
|
|
"request_id": 14,
|
|
"tpot_ms": 106.38107407728566,
|
|
"ttft_ms": 327.5932762319158
|
|
},
|
|
{
|
|
"e2e_ms": 13901.52575941027,
|
|
"request_id": 15,
|
|
"tpot_ms": 106.39141617135505,
|
|
"ttft_ms": 389.81590564817736
|
|
},
|
|
{
|
|
"e2e_ms": 13865.807249012967,
|
|
"request_id": 16,
|
|
"tpot_ms": 106.39141617135505,
|
|
"ttft_ms": 354.097395250875
|
|
},
|
|
{
|
|
"e2e_ms": 13830.088738615665,
|
|
"request_id": 17,
|
|
"tpot_ms": 106.3928821483211,
|
|
"ttft_ms": 318.19270577888403
|
|
},
|
|
{
|
|
"e2e_ms": 13893.62481397874,
|
|
"request_id": 18,
|
|
"tpot_ms": 106.40322424239051,
|
|
"ttft_ms": 380.4153351951456
|
|
},
|
|
{
|
|
"e2e_ms": 13857.906303581438,
|
|
"request_id": 19,
|
|
"tpot_ms": 106.40322424239051,
|
|
"ttft_ms": 344.69682479784325
|
|
},
|
|
{
|
|
"e2e_ms": 13822.187793184135,
|
|
"request_id": 20,
|
|
"tpot_ms": 106.40469021935657,
|
|
"ttft_ms": 308.79213532585226
|
|
},
|
|
{
|
|
"e2e_ms": 13884.410422593844,
|
|
"request_id": 21,
|
|
"tpot_ms": 106.39288214826952,
|
|
"ttft_ms": 372.51438976361584
|
|
},
|
|
{
|
|
"e2e_ms": 13848.691912190276,
|
|
"request_id": 22,
|
|
"tpot_ms": 106.39288214822018,
|
|
"ttft_ms": 336.7958793663135
|
|
},
|
|
{
|
|
"e2e_ms": 13812.78722271334,
|
|
"request_id": 23,
|
|
"tpot_ms": 106.39288214818123,
|
|
"ttft_ms": 300.8911898943225
|
|
},
|
|
{
|
|
"e2e_ms": 13875.009852123696,
|
|
"request_id": 24,
|
|
"tpot_ms": 106.38107407709929,
|
|
"ttft_ms": 364.6134443320861
|
|
},
|
|
{
|
|
"e2e_ms": 13839.291341720127,
|
|
"request_id": 25,
|
|
"tpot_ms": 106.38107407704995,
|
|
"ttft_ms": 328.89493393478375
|
|
},
|
|
{
|
|
"e2e_ms": 13902.827417078257,
|
|
"request_id": 26,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 391.11756334850156
|
|
},
|
|
{
|
|
"e2e_ms": 13867.108906674688,
|
|
"request_id": 27,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 355.39905294493224
|
|
},
|
|
{
|
|
"e2e_ms": 13831.390396271117,
|
|
"request_id": 28,
|
|
"tpot_ms": 106.39288214805607,
|
|
"ttft_ms": 319.49436346799587
|
|
},
|
|
{
|
|
"e2e_ms": 13894.926471629247,
|
|
"request_id": 29,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 381.7169928783528
|
|
},
|
|
{
|
|
"e2e_ms": 13859.207961225678,
|
|
"request_id": 30,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 345.9984824747835
|
|
},
|
|
{
|
|
"e2e_ms": 13823.48945082211,
|
|
"request_id": 31,
|
|
"tpot_ms": 106.40469021908869,
|
|
"ttft_ms": 310.0937929978471
|
|
},
|
|
{
|
|
"e2e_ms": 13885.712080232466,
|
|
"request_id": 32,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 373.8160474293437
|
|
},
|
|
{
|
|
"e2e_ms": 13849.993569828897,
|
|
"request_id": 33,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 338.0975370257744
|
|
},
|
|
{
|
|
"e2e_ms": 13814.08888035196,
|
|
"request_id": 34,
|
|
"tpot_ms": 106.39288214805607,
|
|
"ttft_ms": 302.192847548838
|
|
},
|
|
{
|
|
"e2e_ms": 13876.311509762318,
|
|
"request_id": 35,
|
|
"tpot_ms": 106.3810740770235,
|
|
"ttft_ms": 365.9151019803346
|
|
},
|
|
{
|
|
"e2e_ms": 13840.592999358747,
|
|
"request_id": 36,
|
|
"tpot_ms": 106.38107407702348,
|
|
"ttft_ms": 330.1965915767653
|
|
},
|
|
{
|
|
"e2e_ms": 13904.129074716877,
|
|
"request_id": 37,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 392.41922098712223
|
|
},
|
|
{
|
|
"e2e_ms": 13868.410564313308,
|
|
"request_id": 38,
|
|
"tpot_ms": 106.39141617110043,
|
|
"ttft_ms": 356.7007105835529
|
|
},
|
|
{
|
|
"e2e_ms": 13832.69205390974,
|
|
"request_id": 39,
|
|
"tpot_ms": 106.39288214805609,
|
|
"ttft_ms": 320.79602110661654
|
|
},
|
|
{
|
|
"e2e_ms": 13896.228129267869,
|
|
"request_id": 40,
|
|
"tpot_ms": 106.40322424213304,
|
|
"ttft_ms": 383.0186505169735
|
|
},
|
|
{
|
|
"e2e_ms": 13860.509618864298,
|
|
"request_id": 41,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 347.30014011340415
|
|
},
|
|
{
|
|
"e2e_ms": 13824.79110846073,
|
|
"request_id": 42,
|
|
"tpot_ms": 106.40469021908868,
|
|
"ttft_ms": 311.3954506364678
|
|
},
|
|
{
|
|
"e2e_ms": 13888.32718381886,
|
|
"request_id": 43,
|
|
"tpot_ms": 106.40322424213302,
|
|
"ttft_ms": 375.1177050679644
|
|
},
|
|
{
|
|
"e2e_ms": 13852.60867341529,
|
|
"request_id": 44,
|
|
"tpot_ms": 106.40322424213304,
|
|
"ttft_ms": 339.39919466439505
|
|
},
|
|
{
|
|
"e2e_ms": 13816.89016301172,
|
|
"request_id": 45,
|
|
"tpot_ms": 106.40469021908868,
|
|
"ttft_ms": 303.4945051874587
|
|
},
|
|
{
|
|
"e2e_ms": 13879.11279242067,
|
|
"request_id": 46,
|
|
"tpot_ms": 106.392882148045,
|
|
"ttft_ms": 367.2167596189553
|
|
},
|
|
{
|
|
"e2e_ms": 13843.394282013549,
|
|
"request_id": 47,
|
|
"tpot_ms": 106.39288214801702,
|
|
"ttft_ms": 331.49824921538595
|
|
},
|
|
{
|
|
"e2e_ms": 13807.489592530403,
|
|
"request_id": 48,
|
|
"tpot_ms": 106.39288214796812,
|
|
"ttft_ms": 295.5935597384496
|
|
},
|
|
{
|
|
"e2e_ms": 13869.712221935884,
|
|
"request_id": 49,
|
|
"tpot_ms": 106.38107407689715,
|
|
"ttft_ms": 359.3158141699462
|
|
},
|
|
{
|
|
"e2e_ms": 13833.993711528763,
|
|
"request_id": 50,
|
|
"tpot_ms": 106.38107407686918,
|
|
"ttft_ms": 323.59730376637685
|
|
},
|
|
{
|
|
"e2e_ms": 13897.5297868798,
|
|
"request_id": 51,
|
|
"tpot_ms": 106.3914161708903,
|
|
"ttft_ms": 385.8199331767338
|
|
},
|
|
{
|
|
"e2e_ms": 13861.81127647268,
|
|
"request_id": 52,
|
|
"tpot_ms": 106.39141617089378,
|
|
"ttft_ms": 350.1014227691712
|
|
},
|
|
{
|
|
"e2e_ms": 13826.092766065556,
|
|
"request_id": 53,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 314.1967332860247
|
|
},
|
|
{
|
|
"e2e_ms": 13889.628841416596,
|
|
"request_id": 54,
|
|
"tpot_ms": 106.40322424192983,
|
|
"ttft_ms": 376.41936269150733
|
|
},
|
|
{
|
|
"e2e_ms": 13853.910331009472,
|
|
"request_id": 55,
|
|
"tpot_ms": 106.4032242419298,
|
|
"ttft_ms": 340.7008522843853
|
|
},
|
|
{
|
|
"e2e_ms": 13818.191820602351,
|
|
"request_id": 56,
|
|
"tpot_ms": 106.4046902189064,
|
|
"ttft_ms": 304.7961628012388
|
|
},
|
|
{
|
|
"e2e_ms": 13880.414450007833,
|
|
"request_id": 57,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 368.5184172283016
|
|
},
|
|
{
|
|
"e2e_ms": 13844.695939600711,
|
|
"request_id": 58,
|
|
"tpot_ms": 106.39288214787034,
|
|
"ttft_ms": 332.79990682117955
|
|
},
|
|
{
|
|
"e2e_ms": 13600.134841430987,
|
|
"request_id": 59,
|
|
"tpot_ms": 104.74991829994453,
|
|
"ttft_ms": 296.89521733803304
|
|
},
|
|
{
|
|
"e2e_ms": 13451.749583806417,
|
|
"request_id": 60,
|
|
"tpot_ms": 103.07978040977419,
|
|
"ttft_ms": 360.61747176509584
|
|
},
|
|
{
|
|
"e2e_ms": 13168.506714577745,
|
|
"request_id": 61,
|
|
"tpot_ms": 101.13076971039189,
|
|
"ttft_ms": 324.8989613579738
|
|
},
|
|
{
|
|
"e2e_ms": 12970.21046401943,
|
|
"request_id": 62,
|
|
"tpot_ms": 99.07943994689744,
|
|
"ttft_ms": 387.1215907634564
|
|
},
|
|
{
|
|
"e2e_ms": 12659.030247905519,
|
|
"request_id": 63,
|
|
"tpot_ms": 96.91045013818255,
|
|
"ttft_ms": 351.4030803563344
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "021442528b6c4067d63827496b66b6f530adbe9f4b29732b1aa7aab2112ced9f"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 145.18082785606384,
|
|
"offered_request_rate": 0.8,
|
|
"request_metrics_sha256": "6b246d56546536e46b4e8adfb23076b3cbf053da4b0af0288346df10122d6022",
|
|
"request_rate_per_gpu": 0.1,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 15423.90313583537,
|
|
"request_id": 0,
|
|
"tpot_ms": 119.15146796140039,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 15680.782114004433,
|
|
"request_id": 1,
|
|
"tpot_ms": 120.47488426274414,
|
|
"ttft_ms": 380.47181263592745
|
|
},
|
|
{
|
|
"e2e_ms": 15824.99160915063,
|
|
"request_id": 2,
|
|
"tpot_ms": 121.60222806639713,
|
|
"ttft_ms": 381.5086447181937
|
|
},
|
|
{
|
|
"e2e_ms": 15964.925902693089,
|
|
"request_id": 3,
|
|
"tpot_ms": 122.6571852387084,
|
|
"ttft_ms": 387.4633773771219
|
|
},
|
|
{
|
|
"e2e_ms": 15994.3969757696,
|
|
"request_id": 4,
|
|
"tpot_ms": 123.48492728290344,
|
|
"ttft_ms": 311.8112108408626
|
|
},
|
|
{
|
|
"e2e_ms": 16132.273068839513,
|
|
"request_id": 5,
|
|
"tpot_ms": 124.38017511477935,
|
|
"ttft_ms": 335.9908292625358
|
|
},
|
|
{
|
|
"e2e_ms": 16268.09096143688,
|
|
"request_id": 6,
|
|
"tpot_ms": 125.15372351796954,
|
|
"ttft_ms": 373.5680746547478
|
|
},
|
|
{
|
|
"e2e_ms": 16291.239371010352,
|
|
"request_id": 7,
|
|
"tpot_ms": 125.67763083904141,
|
|
"ttft_ms": 330.18025445209173
|
|
},
|
|
{
|
|
"e2e_ms": 16612.993923996273,
|
|
"request_id": 8,
|
|
"tpot_ms": 127.71634872045071,
|
|
"ttft_ms": 393.0176364990334
|
|
},
|
|
{
|
|
"e2e_ms": 16636.356874836954,
|
|
"request_id": 9,
|
|
"tpot_ms": 128.11558123152213,
|
|
"ttft_ms": 365.67805843364454
|
|
},
|
|
{
|
|
"e2e_ms": 16659.624174048608,
|
|
"request_id": 10,
|
|
"tpot_ms": 128.3236060799542,
|
|
"ttft_ms": 362.52620189442376
|
|
},
|
|
{
|
|
"e2e_ms": 16683.03953534565,
|
|
"request_id": 11,
|
|
"tpot_ms": 128.34365422438844,
|
|
"ttft_ms": 383.3954488483169
|
|
},
|
|
{
|
|
"e2e_ms": 16598.14552827833,
|
|
"request_id": 12,
|
|
"tpot_ms": 128.20865630197483,
|
|
"ttft_ms": 315.6461779275297
|
|
},
|
|
{
|
|
"e2e_ms": 16621.508479120537,
|
|
"request_id": 13,
|
|
"tpot_ms": 128.0747814081899,
|
|
"ttft_ms": 356.0112402804201
|
|
},
|
|
{
|
|
"e2e_ms": 16644.871429963147,
|
|
"request_id": 14,
|
|
"tpot_ms": 127.97565080664988,
|
|
"ttft_ms": 391.9637775186118
|
|
},
|
|
{
|
|
"e2e_ms": 16559.977422897646,
|
|
"request_id": 15,
|
|
"tpot_ms": 127.91134905305496,
|
|
"ttft_ms": 315.2360931596654
|
|
},
|
|
{
|
|
"e2e_ms": 16583.340373740255,
|
|
"request_id": 16,
|
|
"tpot_ms": 127.86325360239432,
|
|
"ttft_ms": 344.7071662361765
|
|
},
|
|
{
|
|
"e2e_ms": 16606.703324582868,
|
|
"request_id": 17,
|
|
"tpot_ms": 127.81515815173371,
|
|
"ttft_ms": 374.17823931268754
|
|
},
|
|
{
|
|
"e2e_ms": 16630.066275425477,
|
|
"request_id": 18,
|
|
"tpot_ms": 127.7832690040065,
|
|
"ttft_ms": 401.59111191665176
|
|
},
|
|
{
|
|
"e2e_ms": 16354.971144941388,
|
|
"request_id": 19,
|
|
"tpot_ms": 126.2873116642462,
|
|
"ttft_ms": 316.48256358212024
|
|
},
|
|
{
|
|
"e2e_ms": 16568.46874002116,
|
|
"request_id": 20,
|
|
"tpot_ms": 127.78612414854776,
|
|
"ttft_ms": 339.6309731555931
|
|
},
|
|
{
|
|
"e2e_ms": 16591.83169086377,
|
|
"request_id": 21,
|
|
"tpot_ms": 127.78612414856296,
|
|
"ttft_ms": 362.9939239962745
|
|
},
|
|
{
|
|
"e2e_ms": 16615.19464170638,
|
|
"request_id": 22,
|
|
"tpot_ms": 127.78612414857815,
|
|
"ttft_ms": 386.3568748369559
|
|
},
|
|
{
|
|
"e2e_ms": 16339.951449136564,
|
|
"request_id": 23,
|
|
"tpot_ms": 126.28847750749865,
|
|
"ttft_ms": 301.3148056842354
|
|
},
|
|
{
|
|
"e2e_ms": 16553.449044216337,
|
|
"request_id": 24,
|
|
"tpot_ms": 127.78518800972483,
|
|
"ttft_ms": 324.73016698128276
|
|
},
|
|
{
|
|
"e2e_ms": 16576.811995058946,
|
|
"request_id": 25,
|
|
"tpot_ms": 127.78477532898123,
|
|
"ttft_ms": 348.1455282783301
|
|
},
|
|
{
|
|
"e2e_ms": 16600.174945901555,
|
|
"request_id": 26,
|
|
"tpot_ms": 127.78477532898441,
|
|
"ttft_ms": 371.50847912053564
|
|
},
|
|
{
|
|
"e2e_ms": 16623.53789674417,
|
|
"request_id": 27,
|
|
"tpot_ms": 127.78477532898442,
|
|
"ttft_ms": 394.87142996314617
|
|
},
|
|
{
|
|
"e2e_ms": 16348.29470417435,
|
|
"request_id": 28,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 309.97742289764574
|
|
},
|
|
{
|
|
"e2e_ms": 16561.79229925412,
|
|
"request_id": 29,
|
|
"tpot_ms": 127.78308602766822,
|
|
"ttft_ms": 333.34037374025627
|
|
},
|
|
{
|
|
"e2e_ms": 16585.155250096734,
|
|
"request_id": 30,
|
|
"tpot_ms": 127.78308602766825,
|
|
"ttft_ms": 356.7033245828668
|
|
},
|
|
{
|
|
"e2e_ms": 16608.518200939343,
|
|
"request_id": 31,
|
|
"tpot_ms": 127.78308602766823,
|
|
"ttft_ms": 380.06627542547733
|
|
},
|
|
{
|
|
"e2e_ms": 16631.881151781956,
|
|
"request_id": 32,
|
|
"tpot_ms": 127.78360948578973,
|
|
"ttft_ms": 403.36274708666053
|
|
},
|
|
{
|
|
"e2e_ms": 16356.786021297865,
|
|
"request_id": 33,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 318.4687400211601
|
|
},
|
|
{
|
|
"e2e_ms": 16570.283616377637,
|
|
"request_id": 34,
|
|
"tpot_ms": 127.78308602766825,
|
|
"ttft_ms": 341.83169086377063
|
|
},
|
|
{
|
|
"e2e_ms": 16593.646567220247,
|
|
"request_id": 35,
|
|
"tpot_ms": 127.78308602766823,
|
|
"ttft_ms": 365.19464170638116
|
|
},
|
|
{
|
|
"e2e_ms": 16617.00951806286,
|
|
"request_id": 36,
|
|
"tpot_ms": 127.78477532898444,
|
|
"ttft_ms": 388.34305128183644
|
|
},
|
|
{
|
|
"e2e_ms": 16341.766325493041,
|
|
"request_id": 37,
|
|
"tpot_ms": 126.28596284469846,
|
|
"ttft_ms": 303.449044216336
|
|
},
|
|
{
|
|
"e2e_ms": 16555.263920572714,
|
|
"request_id": 38,
|
|
"tpot_ms": 127.78308602766747,
|
|
"ttft_ms": 326.81199505894654
|
|
},
|
|
{
|
|
"e2e_ms": 16578.62687141207,
|
|
"request_id": 39,
|
|
"tpot_ms": 127.78308602764184,
|
|
"ttft_ms": 350.17494590155707
|
|
},
|
|
{
|
|
"e2e_ms": 16601.989822251424,
|
|
"request_id": 40,
|
|
"tpot_ms": 127.78308602761618,
|
|
"ttft_ms": 373.5378967441676
|
|
},
|
|
{
|
|
"e2e_ms": 16625.35277309078,
|
|
"request_id": 41,
|
|
"tpot_ms": 127.78477532890675,
|
|
"ttft_ms": 396.6863063196229
|
|
},
|
|
{
|
|
"e2e_ms": 16350.109580518321,
|
|
"request_id": 42,
|
|
"tpot_ms": 126.2859628446,
|
|
"ttft_ms": 311.79229925412244
|
|
},
|
|
{
|
|
"e2e_ms": 16563.607175594883,
|
|
"request_id": 43,
|
|
"tpot_ms": 127.78308602754448,
|
|
"ttft_ms": 335.15525009673297
|
|
},
|
|
{
|
|
"e2e_ms": 16586.970126434237,
|
|
"request_id": 44,
|
|
"tpot_ms": 127.78308602751885,
|
|
"ttft_ms": 358.5182009393435
|
|
},
|
|
{
|
|
"e2e_ms": 16610.333077273594,
|
|
"request_id": 45,
|
|
"tpot_ms": 127.78308602749323,
|
|
"ttft_ms": 381.88115178195403
|
|
},
|
|
{
|
|
"e2e_ms": 16633.69602811295,
|
|
"request_id": 46,
|
|
"tpot_ms": 127.78360948558907,
|
|
"ttft_ms": 405.17762344313724
|
|
},
|
|
{
|
|
"e2e_ms": 16548.802021044266,
|
|
"request_id": 47,
|
|
"tpot_ms": 127.78360948556401,
|
|
"ttft_ms": 320.2836163776368
|
|
},
|
|
{
|
|
"e2e_ms": 16572.164971883623,
|
|
"request_id": 48,
|
|
"tpot_ms": 127.7836094855384,
|
|
"ttft_ms": 343.64656722024733
|
|
},
|
|
{
|
|
"e2e_ms": 16595.52792272298,
|
|
"request_id": 49,
|
|
"tpot_ms": 127.78360948551277,
|
|
"ttft_ms": 367.00951806285786
|
|
},
|
|
{
|
|
"e2e_ms": 16618.890873562337,
|
|
"request_id": 50,
|
|
"tpot_ms": 127.78529878680334,
|
|
"ttft_ms": 390.15792763831314
|
|
},
|
|
{
|
|
"e2e_ms": 16343.647680989874,
|
|
"request_id": 51,
|
|
"tpot_ms": 126.28648630249732,
|
|
"ttft_ms": 305.26392057271323
|
|
},
|
|
{
|
|
"e2e_ms": 16199.563507361632,
|
|
"request_id": 52,
|
|
"tpot_ms": 124.96800500747688,
|
|
"ttft_ms": 328.6268714120695
|
|
},
|
|
{
|
|
"e2e_ms": 16032.332947106512,
|
|
"request_id": 53,
|
|
"tpot_ms": 123.46726869964634,
|
|
"ttft_ms": 351.9898222514257
|
|
},
|
|
{
|
|
"e2e_ms": 15838.885472588829,
|
|
"request_id": 54,
|
|
"tpot_ms": 121.7600999960476,
|
|
"ttft_ms": 375.35277309078197
|
|
},
|
|
{
|
|
"e2e_ms": 15619.209002299955,
|
|
"request_id": 55,
|
|
"tpot_ms": 119.84809306800307,
|
|
"ttft_ms": 398.5011826635656
|
|
},
|
|
{
|
|
"e2e_ms": 15285.473398871076,
|
|
"request_id": 56,
|
|
"tpot_ms": 117.8887104194976,
|
|
"ttft_ms": 313.60717559488194
|
|
},
|
|
{
|
|
"e2e_ms": 15049.070462905931,
|
|
"request_id": 57,
|
|
"tpot_ms": 115.84330973599758,
|
|
"ttft_ms": 336.9701264342382
|
|
},
|
|
{
|
|
"e2e_ms": 14793.47811125659,
|
|
"request_id": 58,
|
|
"tpot_ms": 113.6468112912047,
|
|
"ttft_ms": 360.3330772735944
|
|
},
|
|
{
|
|
"e2e_ms": 14524.751300129523,
|
|
"request_id": 59,
|
|
"tpot_ms": 111.34689190564231,
|
|
"ttft_ms": 383.69602811295067
|
|
},
|
|
{
|
|
"e2e_ms": 14156.606828520311,
|
|
"request_id": 60,
|
|
"tpot_ms": 109.11657328721294,
|
|
"ttft_ms": 298.802021044267
|
|
},
|
|
{
|
|
"e2e_ms": 13870.414276689999,
|
|
"request_id": 61,
|
|
"tpot_ms": 106.67912838430216,
|
|
"ttft_ms": 322.16497188362325
|
|
},
|
|
{
|
|
"e2e_ms": 13579.457505623892,
|
|
"request_id": 62,
|
|
"tpot_ms": 104.20416994410166,
|
|
"ttft_ms": 345.5279227229795
|
|
},
|
|
{
|
|
"e2e_ms": 13281.118281252404,
|
|
"request_id": 63,
|
|
"tpot_ms": 101.6710819503155,
|
|
"ttft_ms": 368.89087356233574
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.15625,
|
|
"passed": 10
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "9ed2bf69d3193253248833822a357db5130e6ca29282e93e04cc92961dba468d"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 139.77111339569092,
|
|
"offered_request_rate": 1.2,
|
|
"request_metrics_sha256": "dfafc592cb8f7c26725458265ca04cb89642135218d5f80993dfbd6e83fcea9c",
|
|
"request_rate_per_gpu": 0.15,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 17349.6420929535,
|
|
"request_id": 0,
|
|
"tpot_ms": 134.3147668363463,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 17472.38469413562,
|
|
"request_id": 1,
|
|
"tpot_ms": 135.04025977135157,
|
|
"ttft_ms": 322.27170317397037
|
|
},
|
|
{
|
|
"e2e_ms": 17594.570506858923,
|
|
"request_id": 2,
|
|
"tpot_ms": 135.72808492940857,
|
|
"ttft_ms": 357.1037208240346
|
|
},
|
|
{
|
|
"e2e_ms": 17607.2367009099,
|
|
"request_id": 3,
|
|
"tpot_ms": 136.28731175562888,
|
|
"ttft_ms": 298.7481079450336
|
|
},
|
|
{
|
|
"e2e_ms": 17921.914449676293,
|
|
"request_id": 4,
|
|
"tpot_ms": 138.39011926907332,
|
|
"ttft_ms": 346.36930250398024
|
|
},
|
|
{
|
|
"e2e_ms": 17935.406822135243,
|
|
"request_id": 5,
|
|
"tpot_ms": 138.88107821741167,
|
|
"ttft_ms": 297.50988852396
|
|
},
|
|
{
|
|
"e2e_ms": 18057.925904932094,
|
|
"request_id": 6,
|
|
"tpot_ms": 139.39521236971183,
|
|
"ttft_ms": 354.73393397869125
|
|
},
|
|
{
|
|
"e2e_ms": 18071.482157513437,
|
|
"request_id": 7,
|
|
"tpot_ms": 139.75067177492934,
|
|
"ttft_ms": 323.14684209741085
|
|
},
|
|
{
|
|
"e2e_ms": 18194.065120430685,
|
|
"request_id": 8,
|
|
"tpot_ms": 140.142283173617,
|
|
"ttft_ms": 395.99515738132584
|
|
},
|
|
{
|
|
"e2e_ms": 18207.834601589766,
|
|
"request_id": 9,
|
|
"tpot_ms": 140.4257392143267,
|
|
"ttft_ms": 373.76572137027256
|
|
},
|
|
{
|
|
"e2e_ms": 18221.607066695313,
|
|
"request_id": 10,
|
|
"tpot_ms": 140.60137526077196,
|
|
"ttft_ms": 365.232408577274
|
|
},
|
|
{
|
|
"e2e_ms": 18235.379531799863,
|
|
"request_id": 11,
|
|
"tpot_ms": 140.670460527253,
|
|
"ttft_ms": 370.23104483873226
|
|
},
|
|
{
|
|
"e2e_ms": 18249.21289308034,
|
|
"request_id": 12,
|
|
"tpot_ms": 140.64257855984852,
|
|
"ttft_ms": 387.6054159795803
|
|
},
|
|
{
|
|
"e2e_ms": 18263.046254360816,
|
|
"request_id": 13,
|
|
"tpot_ms": 140.69995603545655,
|
|
"ttft_ms": 394.1518378578337
|
|
},
|
|
{
|
|
"e2e_ms": 18276.879615640297,
|
|
"request_id": 14,
|
|
"tpot_ms": 140.75508408264292,
|
|
"ttft_ms": 400.98393714464817
|
|
},
|
|
{
|
|
"e2e_ms": 18181.536918127527,
|
|
"request_id": 15,
|
|
"tpot_ms": 140.80327015717634,
|
|
"ttft_ms": 299.52160816613116
|
|
},
|
|
{
|
|
"e2e_ms": 18195.15705083027,
|
|
"request_id": 16,
|
|
"tpot_ms": 140.84624138271937,
|
|
"ttft_ms": 307.6843952249071
|
|
},
|
|
{
|
|
"e2e_ms": 18208.77718353195,
|
|
"request_id": 17,
|
|
"tpot_ms": 140.87800086343316,
|
|
"ttft_ms": 317.2710738759417
|
|
},
|
|
{
|
|
"e2e_ms": 18222.54666469151,
|
|
"request_id": 18,
|
|
"tpot_ms": 140.90607860065492,
|
|
"ttft_ms": 327.4746824083348
|
|
},
|
|
{
|
|
"e2e_ms": 18236.319129797193,
|
|
"request_id": 19,
|
|
"tpot_ms": 140.91088449209656,
|
|
"ttft_ms": 340.6367993009294
|
|
},
|
|
{
|
|
"e2e_ms": 18250.152491077173,
|
|
"request_id": 20,
|
|
"tpot_ms": 140.8981539798204,
|
|
"ttft_ms": 356.0869356399827
|
|
},
|
|
{
|
|
"e2e_ms": 18263.985852358146,
|
|
"request_id": 21,
|
|
"tpot_ms": 140.89857595079965,
|
|
"ttft_ms": 369.8667066065901
|
|
},
|
|
{
|
|
"e2e_ms": 18277.81921363912,
|
|
"request_id": 22,
|
|
"tpot_ms": 140.90338208286406,
|
|
"ttft_ms": 383.0896891153834
|
|
},
|
|
{
|
|
"e2e_ms": 18291.6525749191,
|
|
"request_id": 23,
|
|
"tpot_ms": 140.91257237601366,
|
|
"ttft_ms": 395.7558831653678
|
|
},
|
|
{
|
|
"e2e_ms": 18305.48593620007,
|
|
"request_id": 24,
|
|
"tpot_ms": 140.9217626691632,
|
|
"ttft_ms": 408.4220772163469
|
|
},
|
|
{
|
|
"e2e_ms": 18210.143238687502,
|
|
"request_id": 25,
|
|
"tpot_ms": 140.92327164841032,
|
|
"ttft_ms": 312.8877393393914
|
|
},
|
|
{
|
|
"e2e_ms": 18223.912719846063,
|
|
"request_id": 26,
|
|
"tpot_ms": 140.92545360667498,
|
|
"ttft_ms": 326.38011179834336
|
|
},
|
|
{
|
|
"e2e_ms": 18237.68518495175,
|
|
"request_id": 27,
|
|
"tpot_ms": 140.92765906057843,
|
|
"ttft_ms": 339.8724842582901
|
|
},
|
|
{
|
|
"e2e_ms": 18251.518546232724,
|
|
"request_id": 28,
|
|
"tpot_ms": 140.92984101884323,
|
|
"ttft_ms": 353.42873683963205
|
|
},
|
|
{
|
|
"e2e_ms": 18265.3519075127,
|
|
"request_id": 29,
|
|
"tpot_ms": 140.93202297710803,
|
|
"ttft_ms": 366.98498941997923
|
|
},
|
|
{
|
|
"e2e_ms": 18279.185268793677,
|
|
"request_id": 30,
|
|
"tpot_ms": 140.93370194229104,
|
|
"ttft_ms": 380.60512212271647
|
|
},
|
|
{
|
|
"e2e_ms": 18293.01863007465,
|
|
"request_id": 31,
|
|
"tpot_ms": 140.9341814397353,
|
|
"ttft_ms": 394.3775872282629
|
|
},
|
|
{
|
|
"e2e_ms": 18306.851991354626,
|
|
"request_id": 32,
|
|
"tpot_ms": 140.93466093717961,
|
|
"ttft_ms": 408.15005233281454
|
|
},
|
|
{
|
|
"e2e_ms": 18211.509293842057,
|
|
"request_id": 33,
|
|
"tpot_ms": 140.933484965084,
|
|
"ttft_ms": 312.956703276388
|
|
},
|
|
{
|
|
"e2e_ms": 18225.278775001614,
|
|
"request_id": 34,
|
|
"tpot_ms": 140.9329819720059,
|
|
"ttft_ms": 326.79006455686374
|
|
},
|
|
{
|
|
"e2e_ms": 18239.11213628159,
|
|
"request_id": 35,
|
|
"tpot_ms": 140.93298197200983,
|
|
"ttft_ms": 340.6234258363447
|
|
},
|
|
{
|
|
"e2e_ms": 18252.945497562563,
|
|
"request_id": 36,
|
|
"tpot_ms": 140.93298197201372,
|
|
"ttft_ms": 354.45678711682046
|
|
},
|
|
{
|
|
"e2e_ms": 18266.778858843536,
|
|
"request_id": 37,
|
|
"tpot_ms": 140.93466093719667,
|
|
"ttft_ms": 368.0769198195577
|
|
},
|
|
{
|
|
"e2e_ms": 18280.612220123516,
|
|
"request_id": 38,
|
|
"tpot_ms": 140.93633990237933,
|
|
"ttft_ms": 381.69705252133923
|
|
},
|
|
{
|
|
"e2e_ms": 18294.44558140449,
|
|
"request_id": 39,
|
|
"tpot_ms": 140.93801886756114,
|
|
"ttft_ms": 395.3171852242221
|
|
},
|
|
{
|
|
"e2e_ms": 18308.27894268546,
|
|
"request_id": 40,
|
|
"tpot_ms": 140.93849836500434,
|
|
"ttft_ms": 409.0896503299106
|
|
},
|
|
{
|
|
"e2e_ms": 18212.936245171895,
|
|
"request_id": 41,
|
|
"tpot_ms": 140.93732239290452,
|
|
"ttft_ms": 313.8963012730187
|
|
},
|
|
{
|
|
"e2e_ms": 18226.708710277584,
|
|
"request_id": 42,
|
|
"tpot_ms": 140.93684289546135,
|
|
"ttft_ms": 327.72966255399183
|
|
},
|
|
{
|
|
"e2e_ms": 18047.749752064,
|
|
"request_id": 43,
|
|
"tpot_ms": 139.41879313566167,
|
|
"ttft_ms": 341.56302383496495
|
|
},
|
|
{
|
|
"e2e_ms": 17872.42465139999,
|
|
"request_id": 44,
|
|
"tpot_ms": 137.92935642744132,
|
|
"ttft_ms": 355.3963851149433
|
|
},
|
|
{
|
|
"e2e_ms": 17695.178674250878,
|
|
"request_id": 45,
|
|
"tpot_ms": 136.4247947075194,
|
|
"ttft_ms": 369.22974639591644
|
|
},
|
|
{
|
|
"e2e_ms": 17514.86814366097,
|
|
"request_id": 46,
|
|
"tpot_ms": 134.89610264554392,
|
|
"ttft_ms": 383.06310767688956
|
|
},
|
|
{
|
|
"e2e_ms": 17334.24324324006,
|
|
"request_id": 47,
|
|
"tpot_ms": 133.36661419575813,
|
|
"ttft_ms": 396.68324037877767
|
|
},
|
|
{
|
|
"e2e_ms": 17152.122611965817,
|
|
"request_id": 48,
|
|
"tpot_ms": 131.8241488699319,
|
|
"ttft_ms": 410.4557054844662
|
|
},
|
|
{
|
|
"e2e_ms": 16860.906258226023,
|
|
"request_id": 49,
|
|
"tpot_ms": 130.28066064407446,
|
|
"ttft_ms": 315.26235642856903
|
|
},
|
|
{
|
|
"e2e_ms": 16678.317507592936,
|
|
"request_id": 50,
|
|
"tpot_ms": 128.73402984160936,
|
|
"ttft_ms": 329.0957177085474
|
|
},
|
|
{
|
|
"e2e_ms": 16495.414294222966,
|
|
"request_id": 51,
|
|
"tpot_ms": 127.18492295459406,
|
|
"ttft_ms": 342.9290789895205
|
|
},
|
|
{
|
|
"e2e_ms": 16325.716031493954,
|
|
"request_id": 52,
|
|
"tpot_ms": 125.73979205687765,
|
|
"ttft_ms": 356.76244027049364
|
|
},
|
|
{
|
|
"e2e_ms": 16142.044362006885,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.18463433430247,
|
|
"ttft_ms": 370.595801550472
|
|
},
|
|
{
|
|
"e2e_ms": 15942.642543961227,
|
|
"request_id": 54,
|
|
"tpot_ms": 122.50561717425025,
|
|
"ttft_ms": 384.4291628314451
|
|
},
|
|
{
|
|
"e2e_ms": 15727.503328455541,
|
|
"request_id": 55,
|
|
"tpot_ms": 120.70436246394655,
|
|
"ttft_ms": 398.049295534328
|
|
},
|
|
{
|
|
"e2e_ms": 15403.205770995846,
|
|
"request_id": 56,
|
|
"tpot_ms": 118.90039231904261,
|
|
"ttft_ms": 302.8559464774361
|
|
},
|
|
{
|
|
"e2e_ms": 15178.030676088234,
|
|
"request_id": 57,
|
|
"tpot_ms": 117.01843597110097,
|
|
"ttft_ms": 316.6893077584092
|
|
},
|
|
{
|
|
"e2e_ms": 14941.341931765628,
|
|
"request_id": 58,
|
|
"tpot_ms": 115.04582096634839,
|
|
"ttft_ms": 330.5226690393823
|
|
},
|
|
{
|
|
"e2e_ms": 14696.772511761146,
|
|
"request_id": 59,
|
|
"tpot_ms": 113.01115339717941,
|
|
"ttft_ms": 344.3560303193607
|
|
},
|
|
{
|
|
"e2e_ms": 14451.34286402164,
|
|
"request_id": 60,
|
|
"tpot_ms": 110.96971238127013,
|
|
"ttft_ms": 358.1893916003338
|
|
},
|
|
{
|
|
"e2e_ms": 14196.293999590453,
|
|
"request_id": 61,
|
|
"tpot_ms": 108.85252950164681,
|
|
"ttft_ms": 372.02275288130693
|
|
},
|
|
{
|
|
"e2e_ms": 13938.386603616791,
|
|
"request_id": 62,
|
|
"tpot_ms": 106.71283849964966,
|
|
"ttft_ms": 385.8561141612853
|
|
},
|
|
{
|
|
"e2e_ms": 13676.049735660896,
|
|
"request_id": 63,
|
|
"tpot_ms": 104.53992523504405,
|
|
"ttft_ms": 399.4792308103001
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.125,
|
|
"passed": 8
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "3e07b28ac20428d11b8d83064d68561f03804f8adf37111de43398fdd8d94138"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 163.70222067832947,
|
|
"offered_request_rate": 1.6,
|
|
"request_metrics_sha256": "e85085ea5517c1afa9f850cb838a638360d45db695cde8d3c9fef6baf6af44c4",
|
|
"request_rate_per_gpu": 0.2,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 19447.058061600903,
|
|
"request_id": 0,
|
|
"tpot_ms": 150.8298532036487,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 19750.640261300512,
|
|
"request_id": 1,
|
|
"tpot_ms": 152.83450342712686,
|
|
"ttft_ms": 340.65832605540146
|
|
},
|
|
{
|
|
"e2e_ms": 19750.02534103151,
|
|
"request_id": 2,
|
|
"tpot_ms": 153.17602578595117,
|
|
"ttft_ms": 296.67006621570977
|
|
},
|
|
{
|
|
"e2e_ms": 19855.93358266601,
|
|
"request_id": 3,
|
|
"tpot_ms": 153.5864661294274,
|
|
"ttft_ms": 350.4523842287295
|
|
},
|
|
{
|
|
"e2e_ms": 19855.232098385564,
|
|
"request_id": 4,
|
|
"tpot_ms": 153.87573615619343,
|
|
"ttft_ms": 313.01360654900145
|
|
},
|
|
{
|
|
"e2e_ms": 19961.05377600862,
|
|
"request_id": 5,
|
|
"tpot_ms": 154.2244676371592,
|
|
"ttft_ms": 374.54638608940184
|
|
},
|
|
{
|
|
"e2e_ms": 19960.265727716724,
|
|
"request_id": 6,
|
|
"tpot_ms": 154.4718650613629,
|
|
"ttft_ms": 342.33886492363473
|
|
},
|
|
{
|
|
"e2e_ms": 19959.47767942483,
|
|
"request_id": 7,
|
|
"tpot_ms": 154.6679176552935,
|
|
"ttft_ms": 316.6521372025519
|
|
},
|
|
{
|
|
"e2e_ms": 20065.255011109733,
|
|
"request_id": 8,
|
|
"tpot_ms": 154.88778812601134,
|
|
"ttft_ms": 394.50591910629294
|
|
},
|
|
{
|
|
"e2e_ms": 20064.42261687969,
|
|
"request_id": 9,
|
|
"tpot_ms": 155.03615010045385,
|
|
"ttft_ms": 374.8315541220517
|
|
},
|
|
{
|
|
"e2e_ms": 20063.590222649644,
|
|
"request_id": 10,
|
|
"tpot_ms": 155.117974090142,
|
|
"ttft_ms": 363.6075132016101
|
|
},
|
|
{
|
|
"e2e_ms": 20062.7578284196,
|
|
"request_id": 11,
|
|
"tpot_ms": 155.13454318347627,
|
|
"ttft_ms": 360.67084411811567
|
|
},
|
|
{
|
|
"e2e_ms": 20061.925434189554,
|
|
"request_id": 12,
|
|
"tpot_ms": 155.09034115377113,
|
|
"ttft_ms": 365.45210766062206
|
|
},
|
|
{
|
|
"e2e_ms": 20061.09303995951,
|
|
"request_id": 13,
|
|
"tpot_ms": 155.09665427482798,
|
|
"ttft_ms": 363.8179470563543
|
|
},
|
|
{
|
|
"e2e_ms": 20060.260645729464,
|
|
"request_id": 14,
|
|
"tpot_ms": 155.10154332898202,
|
|
"ttft_ms": 362.3646429487479
|
|
},
|
|
{
|
|
"e2e_ms": 20059.42825149942,
|
|
"request_id": 15,
|
|
"tpot_ms": 155.10604675613317,
|
|
"ttft_ms": 360.96031347050683
|
|
},
|
|
{
|
|
"e2e_ms": 20058.595857269374,
|
|
"request_id": 16,
|
|
"tpot_ms": 155.10168661355542,
|
|
"ttft_ms": 360.6816573478344
|
|
},
|
|
{
|
|
"e2e_ms": 20057.76346303933,
|
|
"request_id": 17,
|
|
"tpot_ms": 155.0900405289204,
|
|
"ttft_ms": 361.3283158664373
|
|
},
|
|
{
|
|
"e2e_ms": 20056.931068809285,
|
|
"request_id": 18,
|
|
"tpot_ms": 155.07436184636043,
|
|
"ttft_ms": 362.4871143215085
|
|
},
|
|
{
|
|
"e2e_ms": 20056.09867457924,
|
|
"request_id": 19,
|
|
"tpot_ms": 155.0434256316313,
|
|
"ttft_ms": 365.58361936206654
|
|
},
|
|
{
|
|
"e2e_ms": 20055.26628034948,
|
|
"request_id": 20,
|
|
"tpot_ms": 154.9995151863024,
|
|
"ttft_ms": 370.32785168907446
|
|
},
|
|
{
|
|
"e2e_ms": 20054.433886119732,
|
|
"request_id": 21,
|
|
"tpot_ms": 154.96275487551375,
|
|
"ttft_ms": 374.16401692948574
|
|
},
|
|
{
|
|
"e2e_ms": 20053.601491889985,
|
|
"request_id": 22,
|
|
"tpot_ms": 154.92303029896564,
|
|
"ttft_ms": 378.3766439213494
|
|
},
|
|
{
|
|
"e2e_ms": 20052.76909766024,
|
|
"request_id": 23,
|
|
"tpot_ms": 154.88711594543284,
|
|
"ttft_ms": 382.105372590269
|
|
},
|
|
{
|
|
"e2e_ms": 20051.936703430492,
|
|
"request_id": 24,
|
|
"tpot_ms": 154.8482609975464,
|
|
"ttft_ms": 386.20755674210193
|
|
},
|
|
{
|
|
"e2e_ms": 20051.104309200746,
|
|
"request_id": 25,
|
|
"tpot_ms": 154.85607608193587,
|
|
"ttft_ms": 384.3826467948901
|
|
},
|
|
{
|
|
"e2e_ms": 20050.271914971,
|
|
"request_id": 26,
|
|
"tpot_ms": 154.86595665257255,
|
|
"ttft_ms": 382.2954200942838
|
|
},
|
|
{
|
|
"e2e_ms": 20049.439520741253,
|
|
"request_id": 27,
|
|
"tpot_ms": 154.87441181850545,
|
|
"ttft_ms": 380.3892197910592
|
|
},
|
|
{
|
|
"e2e_ms": 20048.607126511506,
|
|
"request_id": 28,
|
|
"tpot_ms": 154.87585298283847,
|
|
"ttft_ms": 379.37379769102006
|
|
},
|
|
{
|
|
"e2e_ms": 20047.77473228176,
|
|
"request_id": 29,
|
|
"tpot_ms": 154.8790680483812,
|
|
"ttft_ms": 378.13309013734295
|
|
},
|
|
{
|
|
"e2e_ms": 20046.942338052013,
|
|
"request_id": 30,
|
|
"tpot_ms": 154.879633497606,
|
|
"ttft_ms": 377.228883856052
|
|
},
|
|
{
|
|
"e2e_ms": 20046.109943822266,
|
|
"request_id": 31,
|
|
"tpot_ms": 154.8807461594705,
|
|
"ttft_ms": 376.2551815695119
|
|
},
|
|
{
|
|
"e2e_ms": 20045.27754959252,
|
|
"request_id": 32,
|
|
"tpot_ms": 154.8790337660788,
|
|
"ttft_ms": 375.64026130051076
|
|
},
|
|
{
|
|
"e2e_ms": 19846.84310747305,
|
|
"request_id": 33,
|
|
"tpot_ms": 153.32139973576017,
|
|
"ttft_ms": 375.02534103150964
|
|
},
|
|
{
|
|
"e2e_ms": 19648.543544766126,
|
|
"request_id": 34,
|
|
"tpot_ms": 151.76482774805996,
|
|
"ttft_ms": 374.4104207625085
|
|
},
|
|
{
|
|
"e2e_ms": 19450.677394587543,
|
|
"request_id": 35,
|
|
"tpot_ms": 150.2123500638227,
|
|
"ttft_ms": 373.7089364820605
|
|
},
|
|
{
|
|
"e2e_ms": 19253.237256186894,
|
|
"request_id": 36,
|
|
"tpot_ms": 148.66322680303372,
|
|
"ttft_ms": 373.00745220161247
|
|
},
|
|
{
|
|
"e2e_ms": 19054.80482620875,
|
|
"request_id": 37,
|
|
"tpot_ms": 147.10697182912625,
|
|
"ttft_ms": 372.2194039097175
|
|
},
|
|
{
|
|
"e2e_ms": 18856.422732154002,
|
|
"request_id": 38,
|
|
"tpot_ms": 145.55111320107227,
|
|
"ttft_ms": 371.4313556178226
|
|
},
|
|
{
|
|
"e2e_ms": 18658.616312962055,
|
|
"request_id": 39,
|
|
"tpot_ms": 143.99978744595376,
|
|
"ttft_ms": 370.64330732592765
|
|
},
|
|
{
|
|
"e2e_ms": 18469.244245876795,
|
|
"request_id": 40,
|
|
"tpot_ms": 142.51522309276308,
|
|
"ttft_ms": 369.81091309588265
|
|
},
|
|
{
|
|
"e2e_ms": 18279.907702313452,
|
|
"request_id": 41,
|
|
"tpot_ms": 141.0309384523434,
|
|
"ttft_ms": 368.97851886583766
|
|
},
|
|
{
|
|
"e2e_ms": 18091.61193748763,
|
|
"request_id": 42,
|
|
"tpot_ms": 139.55484892009318,
|
|
"ttft_ms": 368.14612463579266
|
|
},
|
|
{
|
|
"e2e_ms": 17903.060652176137,
|
|
"request_id": 43,
|
|
"tpot_ms": 138.07674741551486,
|
|
"ttft_ms": 367.31373040574766
|
|
},
|
|
{
|
|
"e2e_ms": 17716.73280728946,
|
|
"request_id": 44,
|
|
"tpot_ms": 136.61615331585634,
|
|
"ttft_ms": 366.48133617570267
|
|
},
|
|
{
|
|
"e2e_ms": 17529.12437807872,
|
|
"request_id": 45,
|
|
"tpot_ms": 135.14547587506345,
|
|
"ttft_ms": 365.64894194565767
|
|
},
|
|
{
|
|
"e2e_ms": 17339.472913240785,
|
|
"request_id": 46,
|
|
"tpot_ms": 133.65871153956826,
|
|
"ttft_ms": 364.81654771561267
|
|
},
|
|
{
|
|
"e2e_ms": 17149.611868516175,
|
|
"request_id": 47,
|
|
"tpot_ms": 132.17029696874496,
|
|
"ttft_ms": 363.9841534855677
|
|
},
|
|
{
|
|
"e2e_ms": 16958.75366988868,
|
|
"request_id": 48,
|
|
"tpot_ms": 130.67403079238707,
|
|
"ttft_ms": 363.1517592555227
|
|
},
|
|
{
|
|
"e2e_ms": 16767.44725356325,
|
|
"request_id": 49,
|
|
"tpot_ms": 129.1742353428171,
|
|
"ttft_ms": 362.3193650254777
|
|
},
|
|
{
|
|
"e2e_ms": 16576.276975363853,
|
|
"request_id": 50,
|
|
"tpot_ms": 127.67551184699543,
|
|
"ttft_ms": 361.4869707954327
|
|
},
|
|
{
|
|
"e2e_ms": 16384.89705533921,
|
|
"request_id": 51,
|
|
"tpot_ms": 126.1751376281405,
|
|
"ttft_ms": 360.65457656536637
|
|
},
|
|
{
|
|
"e2e_ms": 16202.320435741867,
|
|
"request_id": 52,
|
|
"tpot_ms": 124.74408073548226,
|
|
"ttft_ms": 359.8221823356198
|
|
},
|
|
{
|
|
"e2e_ms": 16010.428211639819,
|
|
"request_id": 53,
|
|
"tpot_ms": 123.23967262625155,
|
|
"ttft_ms": 358.98978810587323
|
|
},
|
|
{
|
|
"e2e_ms": 15808.049221831383,
|
|
"request_id": 54,
|
|
"tpot_ms": 121.6526915587028,
|
|
"ttft_ms": 358.15739387612666
|
|
},
|
|
{
|
|
"e2e_ms": 15595.17863371626,
|
|
"request_id": 55,
|
|
"tpot_ms": 119.98309948086519,
|
|
"ttft_ms": 357.3249996463801
|
|
},
|
|
{
|
|
"e2e_ms": 15377.4072544157,
|
|
"request_id": 56,
|
|
"tpot_ms": 118.27491849605565,
|
|
"ttft_ms": 356.4926054166335
|
|
},
|
|
{
|
|
"e2e_ms": 15157.846080032627,
|
|
"request_id": 57,
|
|
"tpot_ms": 116.55264463658062,
|
|
"ttft_ms": 355.66021118688695
|
|
},
|
|
{
|
|
"e2e_ms": 14930.609139372891,
|
|
"request_id": 58,
|
|
"tpot_ms": 114.76993167256497,
|
|
"ttft_ms": 354.8278169571404
|
|
},
|
|
{
|
|
"e2e_ms": 14698.1184149259,
|
|
"request_id": 59,
|
|
"tpot_ms": 112.9458503322717,
|
|
"ttft_ms": 353.9954227273938
|
|
},
|
|
{
|
|
"e2e_ms": 14465.054205320868,
|
|
"request_id": 60,
|
|
"tpot_ms": 111.11725336081277,
|
|
"ttft_ms": 353.16302849764725
|
|
},
|
|
{
|
|
"e2e_ms": 14225.577184589198,
|
|
"request_id": 61,
|
|
"tpot_ms": 109.23816181355352,
|
|
"ttft_ms": 352.3306342679007
|
|
},
|
|
{
|
|
"e2e_ms": 13984.194476163324,
|
|
"request_id": 62,
|
|
"tpot_ms": 107.34406485137929,
|
|
"ttft_ms": 351.4982400381541
|
|
},
|
|
{
|
|
"e2e_ms": 13739.858786414643,
|
|
"request_id": 63,
|
|
"tpot_ms": 105.42671606776564,
|
|
"ttft_ms": 350.66584580840754
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.140625,
|
|
"passed": 9
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.4375,
|
|
"passed": 28
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": true,
|
|
"pass_rate": 1.0,
|
|
"passed": 64
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "850087527f8ec5b333ae4bb00c5837c85f7dafca77de3d8c80820e071da8dfbb"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 386.0896592140198,
|
|
"offered_request_rate": 2.4,
|
|
"request_metrics_sha256": "c4ae9e9bb2050189ca506883c0c12efd5faecce0ec6249ad641d32f3b9a7f130",
|
|
"request_rate_per_gpu": 0.3,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 25856.75271176486,
|
|
"request_id": 0,
|
|
"tpot_ms": 201.29988981911293,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 26179.284600266838,
|
|
"request_id": 1,
|
|
"tpot_ms": 203.30897363251975,
|
|
"ttft_ms": 359.0449489368311
|
|
},
|
|
{
|
|
"e2e_ms": 25981.87281483019,
|
|
"request_id": 2,
|
|
"tpot_ms": 201.96786246415923,
|
|
"ttft_ms": 331.9542818819672
|
|
},
|
|
{
|
|
"e2e_ms": 25784.35394549511,
|
|
"request_id": 3,
|
|
"tpot_ms": 200.62102535110301,
|
|
"ttft_ms": 305.48372590502584
|
|
},
|
|
{
|
|
"e2e_ms": 25696.083985341404,
|
|
"request_id": 4,
|
|
"tpot_ms": 199.35042104178692,
|
|
"ttft_ms": 378.5805130344673
|
|
},
|
|
{
|
|
"e2e_ms": 25498.25455731707,
|
|
"request_id": 5,
|
|
"tpot_ms": 197.981349219281,
|
|
"ttft_ms": 354.62320646838384
|
|
},
|
|
{
|
|
"e2e_ms": 25300.295013812065,
|
|
"request_id": 6,
|
|
"tpot_ms": 196.59886557857163,
|
|
"ttft_ms": 332.2390853334691
|
|
},
|
|
{
|
|
"e2e_ms": 25102.100957670827,
|
|
"request_id": 7,
|
|
"tpot_ms": 195.19408059762435,
|
|
"ttft_ms": 312.45272177253236
|
|
},
|
|
{
|
|
"e2e_ms": 25012.968951433395,
|
|
"request_id": 8,
|
|
"tpot_ms": 193.83981874911316,
|
|
"ttft_ms": 395.3119702960262
|
|
},
|
|
{
|
|
"e2e_ms": 24814.29543067174,
|
|
"request_id": 9,
|
|
"tpot_ms": 192.41025790813484,
|
|
"ttft_ms": 378.1926763386156
|
|
},
|
|
{
|
|
"e2e_ms": 24615.47335754875,
|
|
"request_id": 10,
|
|
"tpot_ms": 190.95429488392014,
|
|
"ttft_ms": 364.2779072908935
|
|
},
|
|
{
|
|
"e2e_ms": 24416.46039907787,
|
|
"request_id": 11,
|
|
"tpot_ms": 189.47286981272285,
|
|
"ttft_ms": 353.405932862068
|
|
},
|
|
{
|
|
"e2e_ms": 24216.993764693514,
|
|
"request_id": 12,
|
|
"tpot_ms": 187.96377697549212,
|
|
"ttft_ms": 345.59408880601427
|
|
},
|
|
{
|
|
"e2e_ms": 24017.241128749425,
|
|
"request_id": 13,
|
|
"tpot_ms": 186.4682030159872,
|
|
"ttft_ms": 335.7793457190477
|
|
},
|
|
{
|
|
"e2e_ms": 23816.999491057053,
|
|
"request_id": 14,
|
|
"tpot_ms": 184.96817994362138,
|
|
"ttft_ms": 326.0406382171395
|
|
},
|
|
{
|
|
"e2e_ms": 23616.430479744246,
|
|
"request_id": 15,
|
|
"tpot_ms": 183.46465752234423,
|
|
"ttft_ms": 316.41897440652934
|
|
},
|
|
{
|
|
"e2e_ms": 23412.66959473249,
|
|
"request_id": 16,
|
|
"tpot_ms": 181.92890330417333,
|
|
"ttft_ms": 307.69887510247963
|
|
},
|
|
{
|
|
"e2e_ms": 23315.084360363784,
|
|
"request_id": 17,
|
|
"tpot_ms": 180.37041078851854,
|
|
"ttft_ms": 408.042190221928
|
|
},
|
|
{
|
|
"e2e_ms": 23111.095991700124,
|
|
"request_id": 18,
|
|
"tpot_ms": 178.82629774094855,
|
|
"ttft_ms": 400.1561785996568
|
|
},
|
|
{
|
|
"e2e_ms": 22907.001651879975,
|
|
"request_id": 19,
|
|
"tpot_ms": 177.27413055190453,
|
|
"ttft_ms": 393.18707178810007
|
|
},
|
|
{
|
|
"e2e_ms": 22702.9918523004,
|
|
"request_id": 20,
|
|
"tpot_ms": 175.71469647400258,
|
|
"ttft_ms": 387.22540010207496
|
|
},
|
|
{
|
|
"e2e_ms": 22499.024594644914,
|
|
"request_id": 21,
|
|
"tpot_ms": 174.1539791842547,
|
|
"ttft_ms": 381.46923824456815
|
|
},
|
|
{
|
|
"e2e_ms": 22294.98921706886,
|
|
"request_id": 22,
|
|
"tpot_ms": 172.59076723758398,
|
|
"ttft_ms": 375.96177789569475
|
|
},
|
|
{
|
|
"e2e_ms": 22091.055170736552,
|
|
"request_id": 23,
|
|
"tpot_ms": 171.02806584364455,
|
|
"ttft_ms": 370.49080859369354
|
|
},
|
|
{
|
|
"e2e_ms": 21886.829371806663,
|
|
"request_id": 24,
|
|
"tpot_ms": 169.4599867344259,
|
|
"ttft_ms": 365.4110565345743
|
|
},
|
|
{
|
|
"e2e_ms": 21682.517380669786,
|
|
"request_id": 25,
|
|
"tpot_ms": 167.90637996068475,
|
|
"ttft_ms": 358.40712566282207
|
|
},
|
|
{
|
|
"e2e_ms": 21478.40806232245,
|
|
"request_id": 26,
|
|
"tpot_ms": 166.35416808544093,
|
|
"ttft_ms": 351.4287154714495
|
|
},
|
|
{
|
|
"e2e_ms": 21274.47246393178,
|
|
"request_id": 27,
|
|
"tpot_ms": 164.80170050625136,
|
|
"ttft_ms": 344.6564996378552
|
|
},
|
|
{
|
|
"e2e_ms": 21070.655727841604,
|
|
"request_id": 28,
|
|
"tpot_ms": 163.24706150884583,
|
|
"ttft_ms": 338.278916218183
|
|
},
|
|
{
|
|
"e2e_ms": 20866.83848786135,
|
|
"request_id": 29,
|
|
"tpot_ms": 161.69251523328387,
|
|
"ttft_ms": 331.88905323430083
|
|
},
|
|
{
|
|
"e2e_ms": 20663.178771213603,
|
|
"request_id": 30,
|
|
"tpot_ms": 160.1348533340772,
|
|
"ttft_ms": 326.0523977857979
|
|
},
|
|
{
|
|
"e2e_ms": 20459.48769489552,
|
|
"request_id": 31,
|
|
"tpot_ms": 158.57430185728822,
|
|
"ttft_ms": 320.55135901991605
|
|
},
|
|
{
|
|
"e2e_ms": 20255.951788183756,
|
|
"request_id": 32,
|
|
"tpot_ms": 157.01541837295076,
|
|
"ttft_ms": 314.9936548190073
|
|
},
|
|
{
|
|
"e2e_ms": 20052.589009493837,
|
|
"request_id": 33,
|
|
"tpot_ms": 155.45687267778135,
|
|
"ttft_ms": 309.5661794156026
|
|
},
|
|
{
|
|
"e2e_ms": 19955.947670796646,
|
|
"request_id": 34,
|
|
"tpot_ms": 153.8983040696114,
|
|
"ttft_ms": 410.8630539559979
|
|
},
|
|
{
|
|
"e2e_ms": 19752.847929041352,
|
|
"request_id": 35,
|
|
"tpot_ms": 152.3343481355919,
|
|
"ttft_ms": 406.3857158211803
|
|
},
|
|
{
|
|
"e2e_ms": 19549.96119317403,
|
|
"request_id": 36,
|
|
"tpot_ms": 150.77184485827368,
|
|
"ttft_ms": 401.93689617327345
|
|
},
|
|
{
|
|
"e2e_ms": 19346.57831151796,
|
|
"request_id": 37,
|
|
"tpot_ms": 149.20507336304368,
|
|
"ttft_ms": 397.53399441140937
|
|
},
|
|
{
|
|
"e2e_ms": 19143.22059782458,
|
|
"request_id": 38,
|
|
"tpot_ms": 147.63442381272193,
|
|
"ttft_ms": 393.6487736088932
|
|
},
|
|
{
|
|
"e2e_ms": 18940.150721561607,
|
|
"request_id": 39,
|
|
"tpot_ms": 146.06310547094216,
|
|
"ttft_ms": 390.13632675195356
|
|
},
|
|
{
|
|
"e2e_ms": 18741.29802135198,
|
|
"request_id": 40,
|
|
"tpot_ms": 144.5231784918296,
|
|
"ttft_ms": 386.85435288961753
|
|
},
|
|
{
|
|
"e2e_ms": 18542.463082904305,
|
|
"request_id": 41,
|
|
"tpot_ms": 142.98167897793616,
|
|
"ttft_ms": 383.7898527064141
|
|
},
|
|
{
|
|
"e2e_ms": 18344.148533824395,
|
|
"request_id": 42,
|
|
"tpot_ms": 141.4401875845621,
|
|
"ttft_ms": 381.2447105850083
|
|
},
|
|
{
|
|
"e2e_ms": 18145.706224501653,
|
|
"request_id": 43,
|
|
"tpot_ms": 139.89417135289136,
|
|
"ttft_ms": 379.1464626844494
|
|
},
|
|
{
|
|
"e2e_ms": 17948.37563539231,
|
|
"request_id": 44,
|
|
"tpot_ms": 138.3533077037345,
|
|
"ttft_ms": 377.5055570180292
|
|
},
|
|
{
|
|
"e2e_ms": 17750.40475411994,
|
|
"request_id": 45,
|
|
"tpot_ms": 136.80479670989413,
|
|
"ttft_ms": 376.19557196338246
|
|
},
|
|
{
|
|
"e2e_ms": 17551.412355033975,
|
|
"request_id": 46,
|
|
"tpot_ms": 135.2471347218224,
|
|
"ttft_ms": 375.0262453625339
|
|
},
|
|
{
|
|
"e2e_ms": 17352.31516600567,
|
|
"request_id": 47,
|
|
"tpot_ms": 133.6870504253856,
|
|
"ttft_ms": 374.0597619816981
|
|
},
|
|
{
|
|
"e2e_ms": 17152.719400024922,
|
|
"request_id": 48,
|
|
"tpot_ms": 132.11931086353468,
|
|
"ttft_ms": 373.56692035601566
|
|
},
|
|
{
|
|
"e2e_ms": 16952.899525195207,
|
|
"request_id": 49,
|
|
"tpot_ms": 130.53228068835782,
|
|
"ttft_ms": 375.29987777376664
|
|
},
|
|
{
|
|
"e2e_ms": 16753.147719429508,
|
|
"request_id": 50,
|
|
"tpot_ms": 128.94354832701373,
|
|
"ttft_ms": 377.31708189876304
|
|
},
|
|
{
|
|
"e2e_ms": 16553.291092750187,
|
|
"request_id": 51,
|
|
"tpot_ms": 127.34853489391125,
|
|
"ttft_ms": 380.0271612234596
|
|
},
|
|
{
|
|
"e2e_ms": 16357.836116284518,
|
|
"request_id": 52,
|
|
"tpot_ms": 125.77762361641257,
|
|
"ttft_ms": 384.07791700011984
|
|
},
|
|
{
|
|
"e2e_ms": 16157.72333756749,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.16553823707912,
|
|
"ttft_ms": 388.6999814584407
|
|
},
|
|
{
|
|
"e2e_ms": 15952.367175996273,
|
|
"request_id": 54,
|
|
"tpot_ms": 122.51027887411597,
|
|
"ttft_ms": 393.5617589835445
|
|
},
|
|
{
|
|
"e2e_ms": 15741.765215271713,
|
|
"request_id": 55,
|
|
"tpot_ms": 120.81149800475544,
|
|
"ttft_ms": 398.70496866777216
|
|
},
|
|
{
|
|
"e2e_ms": 15528.71285895543,
|
|
"request_id": 56,
|
|
"tpot_ms": 119.08582914503211,
|
|
"ttft_ms": 404.8125575363528
|
|
},
|
|
{
|
|
"e2e_ms": 15314.765605096894,
|
|
"request_id": 57,
|
|
"tpot_ms": 117.34894161957355,
|
|
"ttft_ms": 411.4500194110526
|
|
},
|
|
{
|
|
"e2e_ms": 15096.980468100028,
|
|
"request_id": 58,
|
|
"tpot_ms": 115.58065073324867,
|
|
"ttft_ms": 418.2378249774459
|
|
},
|
|
{
|
|
"e2e_ms": 14778.441120322277,
|
|
"request_id": 59,
|
|
"tpot_ms": 113.87954551567164,
|
|
"ttft_ms": 315.7388398319796
|
|
},
|
|
{
|
|
"e2e_ms": 14557.742348852762,
|
|
"request_id": 60,
|
|
"tpot_ms": 112.08473611057838,
|
|
"ttft_ms": 322.9808628093096
|
|
},
|
|
{
|
|
"e2e_ms": 14333.83717181993,
|
|
"request_id": 61,
|
|
"tpot_ms": 110.26282336485625,
|
|
"ttft_ms": 330.4586044831872
|
|
},
|
|
{
|
|
"e2e_ms": 14108.97915094099,
|
|
"request_id": 62,
|
|
"tpot_ms": 108.43298632152924,
|
|
"ttft_ms": 337.989888106776
|
|
},
|
|
{
|
|
"e2e_ms": 13882.64463939965,
|
|
"request_id": 63,
|
|
"tpot_ms": 106.58813679106942,
|
|
"ttft_ms": 345.9512669338345
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.125,
|
|
"passed": 8
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.421875,
|
|
"passed": 27
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.71875,
|
|
"passed": 46
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "0b9f30d883de83b4af69dd114bd5d2bb974b7fd1f01d45f913bf5049e4b72a86"
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"elapsed_seconds": 400.40727281570435,
|
|
"offered_request_rate": 3.2,
|
|
"request_metrics_sha256": "48ff1d03e0aeb45bb4395dca86e089cdbe1c7de16b8557fbe8f25e8e65c77def",
|
|
"request_rate_per_gpu": 0.4,
|
|
"requests": [
|
|
{
|
|
"e2e_ms": 26482.398681960134,
|
|
"request_id": 0,
|
|
"tpot_ms": 206.22623604112295,
|
|
"ttft_ms": 291.66670473752106
|
|
},
|
|
{
|
|
"e2e_ms": 26389.27725660841,
|
|
"request_id": 1,
|
|
"tpot_ms": 204.89007083645959,
|
|
"ttft_ms": 368.23826037804076
|
|
},
|
|
{
|
|
"e2e_ms": 26186.398050214455,
|
|
"request_id": 2,
|
|
"tpot_ms": 203.43938315354237,
|
|
"ttft_ms": 349.5963897145746
|
|
},
|
|
{
|
|
"e2e_ms": 25983.46530187178,
|
|
"request_id": 3,
|
|
"tpot_ms": 201.98715121187504,
|
|
"ttft_ms": 331.0970979636474
|
|
},
|
|
{
|
|
"e2e_ms": 25780.42643937068,
|
|
"request_id": 4,
|
|
"tpot_ms": 200.53276712329495,
|
|
"ttft_ms": 312.76501471222184
|
|
},
|
|
{
|
|
"e2e_ms": 25577.34505869151,
|
|
"request_id": 5,
|
|
"tpot_ms": 199.0733214464005,
|
|
"ttft_ms": 295.0332349986446
|
|
},
|
|
{
|
|
"e2e_ms": 25483.55218185351,
|
|
"request_id": 6,
|
|
"tpot_ms": 197.69083780569105,
|
|
"ttft_ms": 376.81578053074594
|
|
},
|
|
{
|
|
"e2e_ms": 25280.28848711639,
|
|
"request_id": 7,
|
|
"tpot_ms": 196.22290463044078,
|
|
"ttft_ms": 359.97959905041466
|
|
},
|
|
{
|
|
"e2e_ms": 25076.966707259246,
|
|
"request_id": 8,
|
|
"tpot_ms": 194.75030755695653,
|
|
"ttft_ms": 343.6776475257637
|
|
},
|
|
{
|
|
"e2e_ms": 24873.46328021192,
|
|
"request_id": 9,
|
|
"tpot_ms": 193.27265662306831,
|
|
"ttft_ms": 327.83588908224147
|
|
},
|
|
{
|
|
"e2e_ms": 24669.88557698392,
|
|
"request_id": 10,
|
|
"tpot_ms": 191.78984111033463,
|
|
"ttft_ms": 312.57575597142215
|
|
},
|
|
{
|
|
"e2e_ms": 24466.212431081483,
|
|
"request_id": 11,
|
|
"tpot_ms": 190.30296300954802,
|
|
"ttft_ms": 297.7361288688827
|
|
},
|
|
{
|
|
"e2e_ms": 24370.912463364126,
|
|
"request_id": 12,
|
|
"tpot_ms": 188.79387017231738,
|
|
"ttft_ms": 394.09095147981964
|
|
},
|
|
{
|
|
"e2e_ms": 24166.86947872558,
|
|
"request_id": 13,
|
|
"tpot_ms": 187.2967209572732,
|
|
"ttft_ms": 380.1859171518851
|
|
},
|
|
{
|
|
"e2e_ms": 23962.581993212396,
|
|
"request_id": 14,
|
|
"tpot_ms": 185.79746051386414,
|
|
"ttft_ms": 366.30450795165007
|
|
},
|
|
{
|
|
"e2e_ms": 23758.13082088949,
|
|
"request_id": 15,
|
|
"tpot_ms": 184.29572160562404,
|
|
"ttft_ms": 352.57417697523687
|
|
},
|
|
{
|
|
"e2e_ms": 23552.083711717114,
|
|
"request_id": 16,
|
|
"tpot_ms": 182.7751996506789,
|
|
"ttft_ms": 339.633356080892
|
|
},
|
|
{
|
|
"e2e_ms": 23345.944189149544,
|
|
"request_id": 17,
|
|
"tpot_ms": 181.25255279330477,
|
|
"ttft_ms": 326.8699843998375
|
|
},
|
|
{
|
|
"e2e_ms": 23139.783338151214,
|
|
"request_id": 18,
|
|
"tpot_ms": 179.72694307410856,
|
|
"ttft_ms": 314.461567739424
|
|
},
|
|
{
|
|
"e2e_ms": 22933.56950157464,
|
|
"request_id": 19,
|
|
"tpot_ms": 178.1977153273412,
|
|
"ttft_ms": 302.45965500230557
|
|
},
|
|
{
|
|
"e2e_ms": 22833.726368661068,
|
|
"request_id": 20,
|
|
"tpot_ms": 176.63828124943853,
|
|
"ttft_ms": 400.6646499823745
|
|
},
|
|
{
|
|
"e2e_ms": 22627.576073166823,
|
|
"request_id": 21,
|
|
"tpot_ms": 175.09813975268048,
|
|
"ttft_ms": 390.11232457640244
|
|
},
|
|
{
|
|
"e2e_ms": 22421.391717712297,
|
|
"request_id": 22,
|
|
"tpot_ms": 173.55627478074584,
|
|
"ttft_ms": 379.7448205575735
|
|
},
|
|
{
|
|
"e2e_ms": 22215.258027879143,
|
|
"request_id": 23,
|
|
"tpot_ms": 172.01247264259936,
|
|
"ttft_ms": 369.67400226902566
|
|
},
|
|
{
|
|
"e2e_ms": 22008.978461747698,
|
|
"request_id": 24,
|
|
"tpot_ms": 170.46437149324944,
|
|
"ttft_ms": 360.0032821050201
|
|
},
|
|
{
|
|
"e2e_ms": 21802.65579951293,
|
|
"request_id": 25,
|
|
"tpot_ms": 168.91532250977505,
|
|
"ttft_ms": 350.4098407714995
|
|
},
|
|
{
|
|
"e2e_ms": 21596.434473671965,
|
|
"request_id": 26,
|
|
"tpot_ms": 167.36573728219415,
|
|
"ttft_ms": 340.98583883331025
|
|
},
|
|
{
|
|
"e2e_ms": 21390.300007810118,
|
|
"request_id": 27,
|
|
"tpot_ms": 165.81511332736412,
|
|
"ttft_ms": 331.7806152348748
|
|
},
|
|
{
|
|
"e2e_ms": 21184.224973098888,
|
|
"request_id": 28,
|
|
"tpot_ms": 164.26380332238384,
|
|
"ttft_ms": 322.72195115614187
|
|
},
|
|
{
|
|
"e2e_ms": 20978.149686441437,
|
|
"request_id": 29,
|
|
"tpot_ms": 162.71174941720716,
|
|
"ttft_ms": 313.7575104561279
|
|
},
|
|
{
|
|
"e2e_ms": 20772.153161450715,
|
|
"request_id": 30,
|
|
"tpot_ms": 161.15510654351164,
|
|
"ttft_ms": 305.4546304247392
|
|
},
|
|
{
|
|
"e2e_ms": 20672.62875180034,
|
|
"request_id": 31,
|
|
"tpot_ms": 159.59455506672822,
|
|
"ttft_ms": 404.12025832585744
|
|
},
|
|
{
|
|
"e2e_ms": 20466.69413177714,
|
|
"request_id": 32,
|
|
"tpot_ms": 158.03533915661183,
|
|
"ttft_ms": 396.2060588874419
|
|
},
|
|
{
|
|
"e2e_ms": 20260.84607576572,
|
|
"request_id": 33,
|
|
"tpot_ms": 156.475168525323,
|
|
"ttft_ms": 388.49967304969545
|
|
},
|
|
{
|
|
"e2e_ms": 20055.02118508905,
|
|
"request_id": 34,
|
|
"tpot_ms": 154.9132321630232,
|
|
"ttft_ms": 381.04070038510326
|
|
},
|
|
{
|
|
"e2e_ms": 19849.304647544788,
|
|
"request_id": 35,
|
|
"tpot_ms": 153.3461190070677,
|
|
"ttft_ms": 374.34753364718887
|
|
},
|
|
{
|
|
"e2e_ms": 19643.694612944317,
|
|
"request_id": 36,
|
|
"tpot_ms": 151.77962188577303,
|
|
"ttft_ms": 367.6826334511443
|
|
},
|
|
{
|
|
"e2e_ms": 19437.836505449406,
|
|
"request_id": 37,
|
|
"tpot_ms": 150.210341914627,
|
|
"ttft_ms": 361.1230822917744
|
|
},
|
|
{
|
|
"e2e_ms": 19231.99098193562,
|
|
"request_id": 38,
|
|
"tpot_ms": 148.63640086508497,
|
|
"ttft_ms": 355.16807206983003
|
|
},
|
|
{
|
|
"e2e_ms": 19026.289377137648,
|
|
"request_id": 39,
|
|
"tpot_ms": 147.05985988148976,
|
|
"ttft_ms": 349.6871721884496
|
|
},
|
|
{
|
|
"e2e_ms": 18822.696360365982,
|
|
"request_id": 40,
|
|
"tpot_ms": 145.49844654462956,
|
|
"ttft_ms": 344.3936491980288
|
|
},
|
|
{
|
|
"e2e_ms": 18619.112224474804,
|
|
"request_id": 41,
|
|
"tpot_ms": 143.93654065290042,
|
|
"ttft_ms": 339.17156155644966
|
|
},
|
|
{
|
|
"e2e_ms": 18415.78828326805,
|
|
"request_id": 42,
|
|
"tpot_ms": 142.37219516274803,
|
|
"ttft_ms": 334.51949759905375
|
|
},
|
|
{
|
|
"e2e_ms": 18212.400461939902,
|
|
"request_id": 43,
|
|
"tpot_ms": 140.8040960160447,
|
|
"ttft_ms": 330.280267902225
|
|
},
|
|
{
|
|
"e2e_ms": 18009.568500717836,
|
|
"request_id": 44,
|
|
"tpot_ms": 139.2366051127279,
|
|
"ttft_ms": 326.5196514013962
|
|
},
|
|
{
|
|
"e2e_ms": 17806.416393414805,
|
|
"request_id": 45,
|
|
"tpot_ms": 137.66365486442547,
|
|
"ttft_ms": 323.1322256327704
|
|
},
|
|
{
|
|
"e2e_ms": 17602.75352720573,
|
|
"request_id": 46,
|
|
"tpot_ms": 136.08599255484802,
|
|
"ttft_ms": 319.83247274003014
|
|
},
|
|
{
|
|
"e2e_ms": 17399.03826602459,
|
|
"request_id": 47,
|
|
"tpot_ms": 134.50648843613666,
|
|
"ttft_ms": 316.71423463523496
|
|
},
|
|
{
|
|
"e2e_ms": 17195.073716367715,
|
|
"request_id": 48,
|
|
"tpot_ms": 132.92201961792392,
|
|
"ttft_ms": 313.9772248913761
|
|
},
|
|
{
|
|
"e2e_ms": 16990.997112286357,
|
|
"request_id": 49,
|
|
"tpot_ms": 131.3320936883647,
|
|
"ttft_ms": 311.8212138640413
|
|
},
|
|
{
|
|
"e2e_ms": 16895.41197318666,
|
|
"request_id": 50,
|
|
"tpot_ms": 129.74336132702078,
|
|
"ttft_ms": 418.0050846550216
|
|
},
|
|
{
|
|
"e2e_ms": 16691.3169931805,
|
|
"request_id": 51,
|
|
"tpot_ms": 128.14881090811454,
|
|
"ttft_ms": 416.418007849952
|
|
},
|
|
{
|
|
"e2e_ms": 16489.422838281163,
|
|
"request_id": 52,
|
|
"tpot_ms": 126.56215930365238,
|
|
"ttft_ms": 416.0286067173118
|
|
},
|
|
{
|
|
"e2e_ms": 16285.199782255653,
|
|
"request_id": 53,
|
|
"tpot_ms": 124.95445752714085,
|
|
"ttft_ms": 415.9836763087661
|
|
},
|
|
{
|
|
"e2e_ms": 16078.355034803544,
|
|
"request_id": 54,
|
|
"tpot_ms": 123.3249765236981,
|
|
"ttft_ms": 416.08301629388575
|
|
},
|
|
{
|
|
"e2e_ms": 15868.887387774763,
|
|
"request_id": 55,
|
|
"tpot_ms": 121.67321161824907,
|
|
"ttft_ms": 416.3895122571297
|
|
},
|
|
{
|
|
"e2e_ms": 15658.194542949623,
|
|
"request_id": 56,
|
|
"tpot_ms": 120.00563624201372,
|
|
"ttft_ms": 417.4787402138804
|
|
},
|
|
{
|
|
"e2e_ms": 15447.054249353854,
|
|
"request_id": 57,
|
|
"tpot_ms": 118.33082151865976,
|
|
"ttft_ms": 419.0399164840635
|
|
},
|
|
{
|
|
"e2e_ms": 15233.99501418892,
|
|
"request_id": 58,
|
|
"tpot_ms": 116.64063648866745,
|
|
"ttft_ms": 420.6341801281539
|
|
},
|
|
{
|
|
"e2e_ms": 15019.622333077174,
|
|
"request_id": 59,
|
|
"tpot_ms": 114.93953127109042,
|
|
"ttft_ms": 422.3018616486911
|
|
},
|
|
{
|
|
"e2e_ms": 14805.106280675915,
|
|
"request_id": 60,
|
|
"tpot_ms": 113.23514577866216,
|
|
"ttft_ms": 424.2427667858202
|
|
},
|
|
{
|
|
"e2e_ms": 14588.987025492997,
|
|
"request_id": 61,
|
|
"tpot_ms": 111.5171161341123,
|
|
"ttft_ms": 426.31327646073555
|
|
},
|
|
{
|
|
"e2e_ms": 14372.391348386529,
|
|
"request_id": 62,
|
|
"tpot_ms": 109.79194855942059,
|
|
"ttft_ms": 428.8138813401155
|
|
},
|
|
{
|
|
"e2e_ms": 14155.057425949359,
|
|
"request_id": 63,
|
|
"tpot_ms": 108.05983074662959,
|
|
"ttft_ms": 431.45892112740114
|
|
}
|
|
],
|
|
"slos": {
|
|
"tpot_120ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.109375,
|
|
"passed": 7
|
|
},
|
|
"tpot_150ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.40625,
|
|
"passed": 26
|
|
},
|
|
"tpot_180ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.71875,
|
|
"passed": 46
|
|
},
|
|
"tpot_40ms": {
|
|
"feasible": false,
|
|
"pass_rate": 0.0,
|
|
"passed": 0
|
|
}
|
|
},
|
|
"status": "completed",
|
|
"trace_sha256": "96c4e04e8f133e66449a5567fcde568f27af44bd9f5d1cadeade66b771a8f0f3"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"contract": {
|
|
"input_tokens": 2048,
|
|
"output_tokens": 128,
|
|
"prefix_caching": false,
|
|
"rates": [
|
|
0.1,
|
|
0.2,
|
|
0.4,
|
|
0.8,
|
|
1.2,
|
|
1.6,
|
|
2.4,
|
|
3.2
|
|
],
|
|
"requests_per_anchor": 64,
|
|
"target_pass_rate": 0.95,
|
|
"tpot_slos_ms": [
|
|
40.0,
|
|
120.0,
|
|
150.0,
|
|
180.0
|
|
],
|
|
"ttft_slo_ms": 1256.0
|
|
},
|
|
"frontier": {
|
|
"declared_upstream_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
|
"fingerprint_source": "prefill-grid-v3 frozen run manifest for the same immutable source snapshot",
|
|
"python_and_config_tree_sha256": "172fc7ae19c40e67030208ff488d0d5d90764888ed76a7a543be6037ba62dc11",
|
|
"source": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/Frontier-d9cfeb6-best-effort-v6-batched-lanes-r3"
|
|
},
|
|
"profiles": {
|
|
"coverage": {
|
|
"rows": 1104,
|
|
"standard_decode_by_tp": {
|
|
"4": 81,
|
|
"8": 81
|
|
},
|
|
"true_mixed_by_tp": {
|
|
"4": 108,
|
|
"8": 108
|
|
}
|
|
},
|
|
"files_sha256": {
|
|
"all_reduce": "8812393e3397d4a0380eae123bbad716ef8242cd2db1fab45e13066f595852b1",
|
|
"attention": "3cc6c78171a85adce6f5c9945ba21be76b9912e580f99e3f2063a15d7c21e1cd",
|
|
"closure_manifest": "38b982b265c5a71b91dbe876d4daaec3b0b15785624eecc042f26836014df1f3",
|
|
"linear": "60006d61a84e2976fe81895a4a1800aa722ffe43ea5d6f2b33a8c40ca6084f33",
|
|
"moe": "58b2b117f332d37a7d2550cce46cf9312de7ccdedaccb2b5680f485e02a06f55"
|
|
},
|
|
"root": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/profiles-t0-full-closed-v1"
|
|
},
|
|
"rankings": {
|
|
"tpot_120ms": [
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 0.8,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.2,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 0.8,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.2,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 0.8,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.2,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 0.8,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.2,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 0.4,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.05,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 0.4,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.05,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 0.4,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.05,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 0.4,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.05,
|
|
"upper_censored": false
|
|
}
|
|
],
|
|
"tpot_150ms": [
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.6,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.4,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.6,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.4,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.6,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.4,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.6,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.4,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.2,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.15,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.2,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.15,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.2,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.15,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.2,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.15,
|
|
"upper_censored": false
|
|
}
|
|
],
|
|
"tpot_180ms": [
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 2.4,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.6,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 2.4,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.6,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 2.4,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.6,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 2.4,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.6,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.6,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.2,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.6,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.2,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.6,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.2,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": false,
|
|
"maximum_tested_feasible_request_rate": 1.6,
|
|
"maximum_tested_feasible_request_rate_per_gpu": 0.2,
|
|
"upper_censored": false
|
|
}
|
|
],
|
|
"tpot_40ms": [
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": true,
|
|
"maximum_tested_feasible_request_rate": null,
|
|
"maximum_tested_feasible_request_rate_per_gpu": null,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns128_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": true,
|
|
"maximum_tested_feasible_request_rate": null,
|
|
"maximum_tested_feasible_request_rate_per_gpu": null,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt16384",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": true,
|
|
"maximum_tested_feasible_request_rate": null,
|
|
"maximum_tested_feasible_request_rate_per_gpu": null,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 1,
|
|
"moe_tp": 4,
|
|
"name": "tp4_mns64_mbt8192",
|
|
"num_gpu_blocks": 26101,
|
|
"tp": 4
|
|
},
|
|
"lower_censored": true,
|
|
"maximum_tested_feasible_request_rate": null,
|
|
"maximum_tested_feasible_request_rate_per_gpu": null,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": true,
|
|
"maximum_tested_feasible_request_rate": null,
|
|
"maximum_tested_feasible_request_rate_per_gpu": null,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 128,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns128_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": true,
|
|
"maximum_tested_feasible_request_rate": null,
|
|
"maximum_tested_feasible_request_rate_per_gpu": null,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 16384,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt16384",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": true,
|
|
"maximum_tested_feasible_request_rate": null,
|
|
"maximum_tested_feasible_request_rate_per_gpu": null,
|
|
"upper_censored": false
|
|
},
|
|
{
|
|
"config": {
|
|
"mbt": 8192,
|
|
"mns": 64,
|
|
"moe_ep": 8,
|
|
"moe_tp": 1,
|
|
"name": "tp8_mns64_mbt8192",
|
|
"num_gpu_blocks": 62351,
|
|
"tp": 8
|
|
},
|
|
"lower_censored": true,
|
|
"maximum_tested_feasible_request_rate": null,
|
|
"maximum_tested_feasible_request_rate_per_gpu": null,
|
|
"upper_censored": false
|
|
}
|
|
]
|
|
},
|
|
"runner": {
|
|
"path": "/home/admin/cpfs/wjh/aituner/aituner-fidelity-v1/run_frontier_t0_surface.py",
|
|
"sha256": "cd69c9b8321361c90ca368a86c551b6ddd6bbe7c0a5cbfbb617affa9e783d9df"
|
|
},
|
|
"schema": "frontier-qwen235b-t0-surface-v1",
|
|
"status": "frozen_before_real_surface"
|
|
}
|