gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Evaluate vLLM 0.20 profiles against Frontier

Browse Source
This commit is contained in:
Gahow Wang
2026-07-16 23:59:10 +08:00
parent 008324e70c
commit 76107d3e87
33 changed files with 13973 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
docs/assets/simulator-fidelity/data.json Normal file
View File

@@ -0,0 +1,60 @@
{
"schema": "simulator-fidelity-figure-data-v1",
"objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu",
"qwen30_mixed": {
"sources": {
"real": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z",
"comparison": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json"
},
"configs": [
{"name": "tp1_mns8", "tp": 1, "mns": 8, "real": 2.1, "frontier_profile_only": 1.1, "frontier_calibrated": 1.7166666666666666},
{"name": "tp1_mns16", "tp": 1, "mns": 16, "real": 2.35, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
{"name": "tp1_mns32", "tp": 1, "mns": 32, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
{"name": "tp1_mns64", "tp": 1, "mns": 64, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
{"name": "tp2_mns8", "tp": 2, "mns": 8, "real": 2.275, "frontier_profile_only": 0.0, "frontier_calibrated": 1.7416666666666667},
{"name": "tp2_mns16", "tp": 2, "mns": 16, "real": 2.275, "frontier_profile_only": 1.1916666666666667, "frontier_calibrated": 2.3},
{"name": "tp2_mns32", "tp": 2, "mns": 32, "real": 3.283333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75},
{"name": "tp2_mns64", "tp": 2, "mns": 64, "real": 3.2583333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75},
{"name": "tp4_mns8", "tp": 4, "mns": 8, "real": 1.2833333333333334, "frontier_profile_only": 0.0, "frontier_calibrated": 1.3208333333333333},
{"name": "tp4_mns16", "tp": 4, "mns": 16, "real": 2.441666666666667, "frontier_profile_only": 0.0, "frontier_calibrated": 2.5},
{"name": "tp4_mns32", "tp": 4, "mns": 32, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5},
{"name": "tp4_mns64", "tp": 4, "mns": 64, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5}
],
"profile_only_metrics": {
"kendall_tau_b": 0.0,
"pairwise_exact_sign_accuracy": 0.3787878787878788,
"simulator_top_set": ["tp4_mns32", "tp4_mns64"],
"real_top_set": ["tp2_mns32"],
"top1_regret_worst": 0.25634517766497456
},
"calibrated_metrics": {
"kendall_tau_b": 0.9668009539030813,
"pairwise_exact_sign_accuracy": 0.9393939393939394,
"simulator_top_set": ["tp2_mns32", "tp2_mns64"],
"real_top_set": ["tp2_mns32"],
"top1_regret_best": 0.0,
"top1_regret_worst": 0.0076142131979695165
}
},
"qwen235_prefill": {
"source": "runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json",
"configs": [
{"name": "tp4_mns64_mbt8192", "tp": 4, "mns": 64, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375},
{"name": "tp4_mns128_mbt8192", "tp": 4, "mns": 128, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375},
{"name": "tp4_mns64_mbt16384", "tp": 4, "mns": 64, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625},
{"name": "tp4_mns128_mbt16384", "tp": 4, "mns": 128, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625},
{"name": "tp8_mns64_mbt8192", "tp": 8, "mns": 64, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05},
{"name": "tp8_mns128_mbt8192", "tp": 8, "mns": 128, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05},
{"name": "tp8_mns64_mbt16384", "tp": 8, "mns": 64, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625},
{"name": "tp8_mns128_mbt16384", "tp": 8, "mns": 128, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625}
],
"metrics": {
"spearman_rank_correlation": 0.9486832980505138,
"pairwise_non_tied_accuracy": 1.0,
"comparable_non_tied_pairs": 20,
"simulator_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"],
"real_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"],
"top1_regret_worst": 0.0
}
}
}