Evaluate vLLM 0.20 profiles against Frontier

docs/assets/simulator-fidelity/data.json

@@ -0,0 +1,60 @@
+{
+  "schema": "simulator-fidelity-figure-data-v1",
+  "objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu",
+  "qwen30_mixed": {
+    "sources": {
+      "real": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z",
+      "comparison": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json"
+    },
+    "configs": [
+      {"name": "tp1_mns8", "tp": 1, "mns": 8, "real": 2.1, "frontier_profile_only": 1.1, "frontier_calibrated": 1.7166666666666666},
+      {"name": "tp1_mns16", "tp": 1, "mns": 16, "real": 2.35, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
+      {"name": "tp1_mns32", "tp": 1, "mns": 32, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
+      {"name": "tp1_mns64", "tp": 1, "mns": 64, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
+      {"name": "tp2_mns8", "tp": 2, "mns": 8, "real": 2.275, "frontier_profile_only": 0.0, "frontier_calibrated": 1.7416666666666667},
+      {"name": "tp2_mns16", "tp": 2, "mns": 16, "real": 2.275, "frontier_profile_only": 1.1916666666666667, "frontier_calibrated": 2.3},
+      {"name": "tp2_mns32", "tp": 2, "mns": 32, "real": 3.283333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75},
+      {"name": "tp2_mns64", "tp": 2, "mns": 64, "real": 3.2583333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75},
+      {"name": "tp4_mns8", "tp": 4, "mns": 8, "real": 1.2833333333333334, "frontier_profile_only": 0.0, "frontier_calibrated": 1.3208333333333333},
+      {"name": "tp4_mns16", "tp": 4, "mns": 16, "real": 2.441666666666667, "frontier_profile_only": 0.0, "frontier_calibrated": 2.5},
+      {"name": "tp4_mns32", "tp": 4, "mns": 32, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5},
+      {"name": "tp4_mns64", "tp": 4, "mns": 64, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5}
+    ],
+    "profile_only_metrics": {
+      "kendall_tau_b": 0.0,
+      "pairwise_exact_sign_accuracy": 0.3787878787878788,
+      "simulator_top_set": ["tp4_mns32", "tp4_mns64"],
+      "real_top_set": ["tp2_mns32"],
+      "top1_regret_worst": 0.25634517766497456
+    },
+    "calibrated_metrics": {
+      "kendall_tau_b": 0.9668009539030813,
+      "pairwise_exact_sign_accuracy": 0.9393939393939394,
+      "simulator_top_set": ["tp2_mns32", "tp2_mns64"],
+      "real_top_set": ["tp2_mns32"],
+      "top1_regret_best": 0.0,
+      "top1_regret_worst": 0.0076142131979695165
+    }
+  },
+  "qwen235_prefill": {
+    "source": "runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json",
+    "configs": [
+      {"name": "tp4_mns64_mbt8192", "tp": 4, "mns": 64, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375},
+      {"name": "tp4_mns128_mbt8192", "tp": 4, "mns": 128, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375},
+      {"name": "tp4_mns64_mbt16384", "tp": 4, "mns": 64, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625},
+      {"name": "tp4_mns128_mbt16384", "tp": 4, "mns": 128, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625},
+      {"name": "tp8_mns64_mbt8192", "tp": 8, "mns": 64, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05},
+      {"name": "tp8_mns128_mbt8192", "tp": 8, "mns": 128, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05},
+      {"name": "tp8_mns64_mbt16384", "tp": 8, "mns": 64, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625},
+      {"name": "tp8_mns128_mbt16384", "tp": 8, "mns": 128, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625}
+    ],
+    "metrics": {
+      "spearman_rank_correlation": 0.9486832980505138,
+      "pairwise_non_tied_accuracy": 1.0,
+      "comparable_non_tied_pairs": 20,
+      "simulator_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"],
+      "real_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"],
+      "top1_regret_worst": 0.0
+    }
+  }
+}