gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Evaluate vLLM 0.20 profiles against Frontier

Browse Source
This commit is contained in:
Gahow Wang
2026-07-16 23:59:10 +08:00
parent 008324e70c
commit 76107d3e87
33 changed files with 13973 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

511
runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.json Normal file
View File

@@ -0,0 +1,511 @@
{
"rows": [
{
"cell": "tp1_mns64",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 2.9833333333333334,
"pass_rate_absolute_error": 0.7430167597765364,
"real_feasible": false,
"real_pass_rate": 0.24581005586592178,
"role": "high1",
"sim_feasible": true,
"sim_pass_rate": 0.9888268156424581,
"sim_throughput_req_s_per_gpu": 2.8823345959582403
},
{
"cell": "tp1_mns64",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 2.033333333333333,
"pass_rate_absolute_error": 0.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": true,
"sim_pass_rate": 1.0,
"sim_throughput_req_s_per_gpu": 1.9806287931485522
},
{
"cell": "tp1_mns8",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 2.9833333333333334,
"pass_rate_absolute_error": 0.03910614525139666,
"real_feasible": false,
"real_pass_rate": 0.20670391061452514,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.16759776536312848,
"sim_throughput_req_s_per_gpu": 2.162661799726284
},
{
"cell": "tp1_mns8",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 2.033333333333333,
"pass_rate_absolute_error": 0.024590163934426257,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": true,
"sim_pass_rate": 0.9754098360655737,
"sim_throughput_req_s_per_gpu": 1.9806313066623409
},
{
"cell": "tp2_mns64",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 2.875,
"pass_rate_absolute_error": 0.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "high1",
"sim_feasible": true,
"sim_pass_rate": 1.0,
"sim_throughput_req_s_per_gpu": 2.809583898291986
},
{
"cell": "tp2_mns64",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 1.9583333333333333,
"pass_rate_absolute_error": 0.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": true,
"sim_pass_rate": 1.0,
"sim_throughput_req_s_per_gpu": 1.911740196570019
},
{
"cell": "tp2_mns8",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 2.8,
"pass_rate_absolute_error": 0.047619047619047616,
"real_feasible": false,
"real_pass_rate": 0.20238095238095238,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.15476190476190477,
"sim_throughput_req_s_per_gpu": 2.083217332903509
},
{
"cell": "tp2_mns8",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 1.9083333333333334,
"pass_rate_absolute_error": 0.013100436681222738,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": true,
"sim_pass_rate": 0.9868995633187773,
"sim_throughput_req_s_per_gpu": 1.8637394408880168
},
{
"cell": "tp4_mns16",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 3.125,
"pass_rate_absolute_error": 0.16266666666666663,
"real_feasible": false,
"real_pass_rate": 0.10666666666666667,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.2693333333333333,
"sim_throughput_req_s_per_gpu": 2.6829294600763345
},
{
"cell": "tp4_mns16",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 2.125,
"pass_rate_absolute_error": 0.3803921568627451,
"real_feasible": false,
"real_pass_rate": 0.6196078431372549,
"role": "low1",
"sim_feasible": true,
"sim_pass_rate": 1.0,
"sim_throughput_req_s_per_gpu": 2.0865667413849516
},
{
"cell": "tp4_mns64",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 3.125,
"pass_rate_absolute_error": 0.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "high1",
"sim_feasible": true,
"sim_pass_rate": 1.0,
"sim_throughput_req_s_per_gpu": 3.0718260341342285
},
{
"cell": "tp4_mns64",
"mode": "historical-calibrated",
"offered_req_s_per_gpu": 2.125,
"pass_rate_absolute_error": 0.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": true,
"sim_pass_rate": 1.0,
"sim_throughput_req_s_per_gpu": 2.0967062495193307
},
{
"cell": "tp1_mns64",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 2.9833333333333334,
"pass_rate_absolute_error": 0.0,
"real_feasible": false,
"real_pass_rate": 0.24581005586592178,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.24581005586592178,
"sim_throughput_req_s_per_gpu": 2.82546810606982
},
{
"cell": "tp1_mns64",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 2.033333333333333,
"pass_rate_absolute_error": 0.2622950819672131,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.7377049180327869,
"sim_throughput_req_s_per_gpu": 1.9427723213024963
},
{
"cell": "tp1_mns8",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 2.9833333333333334,
"pass_rate_absolute_error": 0.111731843575419,
"real_feasible": false,
"real_pass_rate": 0.20670391061452514,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.09497206703910614,
"sim_throughput_req_s_per_gpu": 1.5728783582441022
},
{
"cell": "tp1_mns8",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 2.033333333333333,
"pass_rate_absolute_error": 0.8278688524590164,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.1721311475409836,
"sim_throughput_req_s_per_gpu": 1.5629164685645112
},
{
"cell": "tp2_mns64",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 2.875,
"pass_rate_absolute_error": 0.7391304347826086,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.2608695652173913,
"sim_throughput_req_s_per_gpu": 2.7121507155597584
},
{
"cell": "tp2_mns64",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 1.9583333333333333,
"pass_rate_absolute_error": 0.16170212765957448,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.8382978723404255,
"sim_throughput_req_s_per_gpu": 1.849101894293262
},
{
"cell": "tp2_mns8",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 2.8,
"pass_rate_absolute_error": 0.16071428571428573,
"real_feasible": false,
"real_pass_rate": 0.20238095238095238,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.041666666666666664,
"sim_throughput_req_s_per_gpu": 0.9859710483293953
},
{
"cell": "tp2_mns8",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 1.9083333333333334,
"pass_rate_absolute_error": 0.9388646288209607,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.0611353711790393,
"sim_throughput_req_s_per_gpu": 0.9771135206177947
},
{
"cell": "tp4_mns16",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 3.125,
"pass_rate_absolute_error": 0.07866666666666668,
"real_feasible": false,
"real_pass_rate": 0.10666666666666667,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.028,
"sim_throughput_req_s_per_gpu": 0.9604809617478992
},
{
"cell": "tp4_mns16",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 2.125,
"pass_rate_absolute_error": 0.5607843137254902,
"real_feasible": false,
"real_pass_rate": 0.6196078431372549,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.058823529411764705,
"sim_throughput_req_s_per_gpu": 0.9495221159963836
},
{
"cell": "tp4_mns64",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 3.125,
"pass_rate_absolute_error": 0.9973333333333333,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.0026666666666666666,
"sim_throughput_req_s_per_gpu": 2.281094127221267
},
{
"cell": "tp4_mns64",
"mode": "historical-profile-only",
"offered_req_s_per_gpu": 2.125,
"pass_rate_absolute_error": 0.5372549019607843,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.4627450980392157,
"sim_throughput_req_s_per_gpu": 1.9793960278798877
},
{
"cell": "tp1_mns64",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 2.9833333333333334,
"pass_rate_absolute_error": 0.24581005586592178,
"real_feasible": false,
"real_pass_rate": 0.24581005586592178,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 2.074240230678028
},
{
"cell": "tp1_mns64",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 2.033333333333333,
"pass_rate_absolute_error": 1.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 1.6189525827713862
},
{
"cell": "tp1_mns8",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 2.9833333333333334,
"pass_rate_absolute_error": 0.20670391061452514,
"real_feasible": false,
"real_pass_rate": 0.20670391061452514,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 0.5271192714936628
},
{
"cell": "tp1_mns8",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 2.033333333333333,
"pass_rate_absolute_error": 1.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 0.5225514701978402
},
{
"cell": "tp2_mns64",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 2.875,
"pass_rate_absolute_error": 1.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 1.7237810220598582
},
{
"cell": "tp2_mns64",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 1.9583333333333333,
"pass_rate_absolute_error": 1.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 1.608919184557297
},
{
"cell": "tp2_mns8",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 2.8,
"pass_rate_absolute_error": 0.20238095238095238,
"real_feasible": false,
"real_pass_rate": 0.20238095238095238,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 0.39315152956277477
},
{
"cell": "tp2_mns8",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 1.9083333333333334,
"pass_rate_absolute_error": 1.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 0.3894201655965407
},
{
"cell": "tp4_mns16",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 3.125,
"pass_rate_absolute_error": 0.10666666666666667,
"real_feasible": false,
"real_pass_rate": 0.10666666666666667,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 0.43700447014112276
},
{
"cell": "tp4_mns16",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 2.125,
"pass_rate_absolute_error": 0.6196078431372549,
"real_feasible": false,
"real_pass_rate": 0.6196078431372549,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 0.43475717185302865
},
{
"cell": "tp4_mns64",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 3.125,
"pass_rate_absolute_error": 1.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "high1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 1.2598449015844073
},
{
"cell": "tp4_mns64",
"mode": "vllm020-profile-only",
"offered_req_s_per_gpu": 2.125,
"pass_rate_absolute_error": 1.0,
"real_feasible": true,
"real_pass_rate": 1.0,
"role": "low1",
"sim_feasible": false,
"sim_pass_rate": 0.0,
"sim_throughput_req_s_per_gpu": 1.2025831680037373
}
],
"schema": "frontier-qwen30-p1-profile-ablation.v1",
"scope": {
"cells": 6,
"probes_per_cell": 2,
"reading": "held-out boundary classification, not a complete capacity sweep",
"roles": [
"low1",
"high1"
]
},
"sources": {
"controller_state": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/controller-state.json",
"historical_calibrated": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/results/metrics.json",
"historical_profile_only": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/old-profile-only-v2/results/metrics.json",
"vllm020_profile_only": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/new-profile-only-v2/results/metrics.json"
},
"summaries": {
"historical-calibrated": {
"feasible_probe_count": 9,
"p1_capacity_lower_bounds_req_s_per_gpu": {
"tp1_mns64": 2.9833333333333334,
"tp1_mns8": 2.033333333333333,
"tp2_mns64": 2.875,
"tp2_mns8": 1.9083333333333334,
"tp4_mns16": 2.125,
"tp4_mns64": 3.125
},
"pass_rate_mae": 0.11754094806600345,
"probe_classification": {
"accuracy": 0.8333333333333334,
"agreement": 10,
"false_feasible": 2,
"false_infeasible": 0
},
"rank_identifiable": true
},
"historical-profile-only": {
"feasible_probe_count": 0,
"p1_capacity_lower_bounds_req_s_per_gpu": {
"tp1_mns64": 0.0,
"tp1_mns8": 0.0,
"tp2_mns64": 0.0,
"tp2_mns8": 0.0,
"tp4_mns16": 0.0,
"tp4_mns64": 0.0
},
"pass_rate_mae": 0.44802887255544604,
"probe_classification": {
"accuracy": 0.4166666666666667,
"agreement": 5,
"false_feasible": 0,
"false_infeasible": 7
},
"rank_identifiable": false
},
"vllm020-profile-only": {
"feasible_probe_count": 0,
"p1_capacity_lower_bounds_req_s_per_gpu": {
"tp1_mns64": 0.0,
"tp1_mns8": 0.0,
"tp2_mns64": 0.0,
"tp2_mns8": 0.0,
"tp4_mns16": 0.0,
"tp4_mns64": 0.0
},
"pass_rate_mae": 0.69843078572211,
"probe_classification": {
"accuracy": 0.4166666666666667,
"agreement": 5,
"false_feasible": 0,
"false_infeasible": 7
},
"rank_identifiable": false
}
}
}