20 lines
738 B
JSON
20 lines
738 B
JSON
{
|
|
"observation": "Push batching further after validating the balanced layout.",
|
|
"diagnosis": "If TTFT remains under control, a larger admission window should maximize achieved request rate.",
|
|
"config_patch": {
|
|
"env_patch": {},
|
|
"flag_patch": {
|
|
"tensor-parallel-size": 4,
|
|
"max-num-seqs": 24,
|
|
"max-num-batched-tokens": 98304,
|
|
"gpu-memory-utilization": 0.92,
|
|
"block-size": 64
|
|
}
|
|
},
|
|
"expected_effects": [
|
|
"Highest throughput among the smoke candidates if memory is sufficient",
|
|
"More pressure on TTFT, so the binary search should locate the safe threshold"
|
|
],
|
|
"why_not_previous_failures": "Keeps tp=4 and block-size stable while only expanding batching and memory utilization."
|
|
}
|