Bound graph predictor grid to runtime
This commit is contained in:
@@ -454,6 +454,13 @@ def main() -> None:
|
||||
config_knobs["no_cache"] = args.fresh_predictor_cache
|
||||
if args.align_real_graph_runtime:
|
||||
config_knobs["num_blocks"] = REAL_NUM_BLOCKS_BY_CONFIG[(config.tp, config.mns)]
|
||||
# These are predictor-grid limits, not scheduler knobs. Keep the
|
||||
# grid inside the observed vLLM graph/runtime envelope: no batch
|
||||
# can use a capture bucket above this cell's list and MBT is 8192.
|
||||
config_knobs["prediction_max_batch_size"] = max(
|
||||
GRAPH_CAPTURE_SIZES_BY_MNS[config.mns]
|
||||
)
|
||||
config_knobs["prediction_max_prefill_chunk_size"] = 8192
|
||||
if kernel_paths is not None:
|
||||
config_knobs.update(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user