{
  "schema": "simulator-fidelity-figure-data-v1",
  "objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu",
  "qwen30_mixed": {
    "sources": {
      "real": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z",
      "comparison": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json"
    },
    "configs": [
      {"name": "tp1_mns8", "tp": 1, "mns": 8, "real": 2.1, "frontier_profile_only": 1.1, "frontier_calibrated": 1.7166666666666666},
      {"name": "tp1_mns16", "tp": 1, "mns": 16, "real": 2.35, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
      {"name": "tp1_mns32", "tp": 1, "mns": 32, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
      {"name": "tp1_mns64", "tp": 1, "mns": 64, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333},
      {"name": "tp2_mns8", "tp": 2, "mns": 8, "real": 2.275, "frontier_profile_only": 0.0, "frontier_calibrated": 1.7416666666666667},
      {"name": "tp2_mns16", "tp": 2, "mns": 16, "real": 2.275, "frontier_profile_only": 1.1916666666666667, "frontier_calibrated": 2.3},
      {"name": "tp2_mns32", "tp": 2, "mns": 32, "real": 3.283333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75},
      {"name": "tp2_mns64", "tp": 2, "mns": 64, "real": 3.2583333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75},
      {"name": "tp4_mns8", "tp": 4, "mns": 8, "real": 1.2833333333333334, "frontier_profile_only": 0.0, "frontier_calibrated": 1.3208333333333333},
      {"name": "tp4_mns16", "tp": 4, "mns": 16, "real": 2.441666666666667, "frontier_profile_only": 0.0, "frontier_calibrated": 2.5},
      {"name": "tp4_mns32", "tp": 4, "mns": 32, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5},
      {"name": "tp4_mns64", "tp": 4, "mns": 64, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5}
    ],
    "profile_only_metrics": {
      "kendall_tau_b": 0.0,
      "pairwise_exact_sign_accuracy": 0.3787878787878788,
      "simulator_top_set": ["tp4_mns32", "tp4_mns64"],
      "real_top_set": ["tp2_mns32"],
      "top1_regret_worst": 0.25634517766497456
    },
    "calibrated_metrics": {
      "kendall_tau_b": 0.9668009539030813,
      "pairwise_exact_sign_accuracy": 0.9393939393939394,
      "simulator_top_set": ["tp2_mns32", "tp2_mns64"],
      "real_top_set": ["tp2_mns32"],
      "top1_regret_best": 0.0,
      "top1_regret_worst": 0.0076142131979695165
    }
  },
  "qwen235_prefill": {
    "source": "runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json",
    "configs": [
      {"name": "tp4_mns64_mbt8192", "tp": 4, "mns": 64, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375},
      {"name": "tp4_mns128_mbt8192", "tp": 4, "mns": 128, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375},
      {"name": "tp4_mns64_mbt16384", "tp": 4, "mns": 64, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625},
      {"name": "tp4_mns128_mbt16384", "tp": 4, "mns": 128, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625},
      {"name": "tp8_mns64_mbt8192", "tp": 8, "mns": 64, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05},
      {"name": "tp8_mns128_mbt8192", "tp": 8, "mns": 128, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05},
      {"name": "tp8_mns64_mbt16384", "tp": 8, "mns": 64, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625},
      {"name": "tp8_mns128_mbt16384", "tp": 8, "mns": 128, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625}
    ],
    "metrics": {
      "spearman_rank_correlation": 0.9486832980505138,
      "pairwise_non_tied_accuracy": 1.0,
      "comparable_non_tied_pairs": 20,
      "simulator_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"],
      "real_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"],
      "top1_regret_worst": 0.0
    }
  }
}