388 lines
13 KiB
JSON
388 lines
13 KiB
JSON
{
|
|
"aligned_ranking": {
|
|
"frontier_tolerance": 3.7499999999999997e-06,
|
|
"kendall_counts": {
|
|
"both_tied": 4,
|
|
"concordant": 58,
|
|
"frontier_only_tied": 3,
|
|
"real_only_tied": 1
|
|
},
|
|
"kendall_tau_b": 0.9668009539030813,
|
|
"pair_count": 66,
|
|
"pairwise_exact_sign_accuracy": 0.9393939393939394,
|
|
"real_tolerance": 3.283333333333333e-06,
|
|
"top1_candidate_cells": [
|
|
"tp2_mns32",
|
|
"tp2_mns64"
|
|
],
|
|
"top1_effective_k": 2,
|
|
"top1_optimistic_regret": 0.0,
|
|
"top1_worst_case_regret": 0.0076142131979695165
|
|
},
|
|
"anchor_feasibility": {
|
|
"accuracy": 0.6956521739130435,
|
|
"false_feasible": 21,
|
|
"false_feasible_rate": 0.7,
|
|
"false_infeasible": 7,
|
|
"false_infeasible_rate": 0.11290322580645161,
|
|
"feasible_recall": 0.8870967741935484,
|
|
"infeasible_recall": 0.3,
|
|
"total": 92,
|
|
"true_feasible": 55,
|
|
"true_infeasible": 9
|
|
},
|
|
"cells": {
|
|
"tp1_mns16": {
|
|
"absolute_error": 0.033333333333333215,
|
|
"anchor_count": 8,
|
|
"frontier_boundary_bracketed": true,
|
|
"frontier_feasible_anchor_count": 7,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": false,
|
|
"frontier_score": 2.3833333333333333,
|
|
"frontier_selected_probe": 1,
|
|
"max_common_anchor_req_s_per_gpu": 4.6,
|
|
"mns": 16,
|
|
"real_feasible_anchor_count": 6,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.35,
|
|
"real_selected_probe": 7,
|
|
"relative_error": 0.014184397163120517,
|
|
"tp": 1
|
|
},
|
|
"tp1_mns32": {
|
|
"absolute_error": 0.10000000000000009,
|
|
"anchor_count": 8,
|
|
"frontier_boundary_bracketed": true,
|
|
"frontier_feasible_anchor_count": 7,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": false,
|
|
"frontier_score": 2.3833333333333333,
|
|
"frontier_selected_probe": 1,
|
|
"max_common_anchor_req_s_per_gpu": 4.6,
|
|
"mns": 32,
|
|
"real_feasible_anchor_count": 5,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.283333333333333,
|
|
"real_selected_probe": 6,
|
|
"relative_error": 0.04379562043795625,
|
|
"tp": 1
|
|
},
|
|
"tp1_mns64": {
|
|
"absolute_error": 0.10000000000000009,
|
|
"anchor_count": 8,
|
|
"frontier_boundary_bracketed": true,
|
|
"frontier_feasible_anchor_count": 7,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": false,
|
|
"frontier_score": 2.3833333333333333,
|
|
"frontier_selected_probe": 1,
|
|
"max_common_anchor_req_s_per_gpu": 4.6,
|
|
"mns": 64,
|
|
"real_feasible_anchor_count": 5,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.283333333333333,
|
|
"real_selected_probe": 6,
|
|
"relative_error": 0.04379562043795625,
|
|
"tp": 1
|
|
},
|
|
"tp1_mns8": {
|
|
"absolute_error": -0.3833333333333335,
|
|
"anchor_count": 8,
|
|
"frontier_boundary_bracketed": true,
|
|
"frontier_feasible_anchor_count": 2,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": false,
|
|
"frontier_score": 1.7166666666666666,
|
|
"frontier_selected_probe": 3,
|
|
"max_common_anchor_req_s_per_gpu": 4.6,
|
|
"mns": 8,
|
|
"real_feasible_anchor_count": 4,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.1,
|
|
"real_selected_probe": 6,
|
|
"relative_error": -0.1825396825396826,
|
|
"tp": 1
|
|
},
|
|
"tp2_mns16": {
|
|
"absolute_error": 0.02499999999999991,
|
|
"anchor_count": 8,
|
|
"frontier_boundary_bracketed": false,
|
|
"frontier_feasible_anchor_count": 8,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": true,
|
|
"frontier_score": 2.3,
|
|
"frontier_selected_probe": 0,
|
|
"max_common_anchor_req_s_per_gpu": 2.3,
|
|
"mns": 16,
|
|
"real_feasible_anchor_count": 7,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.275,
|
|
"real_selected_probe": 7,
|
|
"relative_error": 0.01098901098901095,
|
|
"tp": 2
|
|
},
|
|
"tp2_mns32": {
|
|
"absolute_error": 0.4666666666666668,
|
|
"anchor_count": 8,
|
|
"frontier_boundary_bracketed": false,
|
|
"frontier_feasible_anchor_count": 8,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": true,
|
|
"frontier_score": 3.75,
|
|
"frontier_selected_probe": 2,
|
|
"max_common_anchor_req_s_per_gpu": 3.75,
|
|
"mns": 32,
|
|
"real_feasible_anchor_count": 3,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 3.283333333333333,
|
|
"real_selected_probe": 7,
|
|
"relative_error": 0.1421319796954315,
|
|
"tp": 2
|
|
},
|
|
"tp2_mns64": {
|
|
"absolute_error": 0.4916666666666667,
|
|
"anchor_count": 8,
|
|
"frontier_boundary_bracketed": false,
|
|
"frontier_feasible_anchor_count": 8,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": true,
|
|
"frontier_score": 3.75,
|
|
"frontier_selected_probe": 2,
|
|
"max_common_anchor_req_s_per_gpu": 3.75,
|
|
"mns": 64,
|
|
"real_feasible_anchor_count": 2,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 3.2583333333333333,
|
|
"real_selected_probe": 1,
|
|
"relative_error": 0.15089514066496165,
|
|
"tp": 2
|
|
},
|
|
"tp2_mns8": {
|
|
"absolute_error": -0.5333333333333332,
|
|
"anchor_count": 8,
|
|
"frontier_boundary_bracketed": true,
|
|
"frontier_feasible_anchor_count": 2,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": false,
|
|
"frontier_score": 1.7416666666666667,
|
|
"frontier_selected_probe": 2,
|
|
"max_common_anchor_req_s_per_gpu": 2.3,
|
|
"mns": 8,
|
|
"real_feasible_anchor_count": 7,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.275,
|
|
"real_selected_probe": 7,
|
|
"relative_error": -0.23443223443223438,
|
|
"tp": 2
|
|
},
|
|
"tp4_mns16": {
|
|
"absolute_error": 0.058333333333333126,
|
|
"anchor_count": 7,
|
|
"frontier_boundary_bracketed": false,
|
|
"frontier_feasible_anchor_count": 7,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": true,
|
|
"frontier_score": 2.5,
|
|
"frontier_selected_probe": 0,
|
|
"max_common_anchor_req_s_per_gpu": 2.5,
|
|
"mns": 16,
|
|
"real_feasible_anchor_count": 6,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.441666666666667,
|
|
"real_selected_probe": 6,
|
|
"relative_error": 0.02389078498293507,
|
|
"tp": 4
|
|
},
|
|
"tp4_mns32": {
|
|
"absolute_error": 0.058333333333333126,
|
|
"anchor_count": 7,
|
|
"frontier_boundary_bracketed": false,
|
|
"frontier_feasible_anchor_count": 7,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": true,
|
|
"frontier_score": 2.5,
|
|
"frontier_selected_probe": 0,
|
|
"max_common_anchor_req_s_per_gpu": 2.5,
|
|
"mns": 32,
|
|
"real_feasible_anchor_count": 6,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.441666666666667,
|
|
"real_selected_probe": 6,
|
|
"relative_error": 0.02389078498293507,
|
|
"tp": 4
|
|
},
|
|
"tp4_mns64": {
|
|
"absolute_error": 0.058333333333333126,
|
|
"anchor_count": 7,
|
|
"frontier_boundary_bracketed": false,
|
|
"frontier_feasible_anchor_count": 7,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": true,
|
|
"frontier_score": 2.5,
|
|
"frontier_selected_probe": 0,
|
|
"max_common_anchor_req_s_per_gpu": 2.5,
|
|
"mns": 64,
|
|
"real_feasible_anchor_count": 6,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 2.441666666666667,
|
|
"real_selected_probe": 6,
|
|
"relative_error": 0.02389078498293507,
|
|
"tp": 4
|
|
},
|
|
"tp4_mns8": {
|
|
"absolute_error": 0.03749999999999987,
|
|
"anchor_count": 7,
|
|
"frontier_boundary_bracketed": true,
|
|
"frontier_feasible_anchor_count": 6,
|
|
"frontier_monotonic_violation_count": 0,
|
|
"frontier_monotonic_violations": [],
|
|
"frontier_right_censored": false,
|
|
"frontier_score": 1.3208333333333333,
|
|
"frontier_selected_probe": 1,
|
|
"max_common_anchor_req_s_per_gpu": 2.5,
|
|
"mns": 8,
|
|
"real_feasible_anchor_count": 5,
|
|
"real_monotonic_violation_count": 0,
|
|
"real_monotonic_violations": [],
|
|
"real_score": 1.2833333333333334,
|
|
"real_selected_probe": 6,
|
|
"relative_error": 0.029220779220779116,
|
|
"tp": 4
|
|
}
|
|
},
|
|
"claim": "On the frozen common anchor grid, rank configurations by maximum offered req/s/GPU whose request-level SLO pass rate is at least 0.95 in each system.",
|
|
"estimand": "paired-grid-slo-feasible-max-offered-throughput",
|
|
"generated_utc": "2026-07-15T07:27:18.082402+00:00",
|
|
"grouped_score_error": {
|
|
"by_mns": {
|
|
"16": {
|
|
"cell_count": 3,
|
|
"mean_absolute_error_req_s_per_gpu": 0.03888888888888875,
|
|
"mean_signed_error_req_s_per_gpu": 0.03888888888888875,
|
|
"right_censored_cell_count": 2
|
|
},
|
|
"32": {
|
|
"cell_count": 3,
|
|
"mean_absolute_error_req_s_per_gpu": 0.20833333333333334,
|
|
"mean_signed_error_req_s_per_gpu": 0.20833333333333334,
|
|
"right_censored_cell_count": 2
|
|
},
|
|
"64": {
|
|
"cell_count": 3,
|
|
"mean_absolute_error_req_s_per_gpu": 0.21666666666666665,
|
|
"mean_signed_error_req_s_per_gpu": 0.21666666666666665,
|
|
"right_censored_cell_count": 2
|
|
},
|
|
"8": {
|
|
"cell_count": 3,
|
|
"mean_absolute_error_req_s_per_gpu": 0.31805555555555554,
|
|
"mean_signed_error_req_s_per_gpu": -0.2930555555555556,
|
|
"right_censored_cell_count": 0
|
|
}
|
|
},
|
|
"by_tp": {
|
|
"1": {
|
|
"cell_count": 4,
|
|
"mean_absolute_error_req_s_per_gpu": 0.15416666666666673,
|
|
"mean_signed_error_req_s_per_gpu": -0.03750000000000003,
|
|
"right_censored_cell_count": 0
|
|
},
|
|
"2": {
|
|
"cell_count": 4,
|
|
"mean_absolute_error_req_s_per_gpu": 0.37916666666666665,
|
|
"mean_signed_error_req_s_per_gpu": 0.11250000000000004,
|
|
"right_censored_cell_count": 3
|
|
},
|
|
"4": {
|
|
"cell_count": 4,
|
|
"mean_absolute_error_req_s_per_gpu": 0.05312499999999981,
|
|
"mean_signed_error_req_s_per_gpu": 0.05312499999999981,
|
|
"right_censored_cell_count": 3
|
|
}
|
|
}
|
|
},
|
|
"independent_reconstruction_crosscheck": {
|
|
"maximum_frontier_score_absolute_difference": 0.0,
|
|
"maximum_real_score_absolute_difference": 0.0,
|
|
"previous_reading": "frozen-calibrated/SLO-gated",
|
|
"status": "PASS"
|
|
},
|
|
"inputs": {
|
|
"frontier_results": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results",
|
|
"ground_truth": {
|
|
"path": "/home/gahow/phd/replayserve/docs/assets/simfid_s2r/ground_truth.json",
|
|
"sha256": "23ff3e6f6b88df8632bf37d37c0ff87bfef9d1676db81592948c08e898f7a670"
|
|
},
|
|
"previous_metrics": {
|
|
"path": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json",
|
|
"sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206"
|
|
}
|
|
},
|
|
"known_limits": [
|
|
"real probes reused a process and prefix-cache history within each cell",
|
|
"Frontier anchors were independent simulator runs",
|
|
"right-censored cells do not identify Frontier's exact capacity knee",
|
|
"the anchor grid is ragged across cells"
|
|
],
|
|
"objective_comparison": {
|
|
"old_proxy": {
|
|
"kendall_tau_b": 0.44812907976513594,
|
|
"pairwise_exact_sign_accuracy": 0.6818181818181818,
|
|
"reading": "completed-throughput proxy (old, not SLO gated)",
|
|
"top1_candidate_cells": [
|
|
"tp1_mns64"
|
|
],
|
|
"top1_worst_case_regret": 0.30456852791878175
|
|
}
|
|
},
|
|
"provenance": {
|
|
"analysis_repository": {
|
|
"head": "9c8570f36b1753d1b77c7cf1b0d16ecc360cad33",
|
|
"status_short": "?? \"AITuner\\347\\263\\273\\347\\273\\237\\344\\274\\230\\345\\214\\226\\344\\270\\216\\346\\214\\221\\346\\210\\230.pdf\"\n?? runs/frontier-slo-alignment-v0/"
|
|
},
|
|
"anchor_count": 92,
|
|
"cell_count": 12,
|
|
"frontier_git_head_recorded_by_runs": "d9cfeb6d8791fbf2f295dd9744c56a666171776e",
|
|
"frontier_mode": "frozen-calibrated",
|
|
"real_engine_version": "vLLM 0.20.0"
|
|
},
|
|
"schema_version": "frontier-slo-alignment-v0",
|
|
"score_error": {
|
|
"bracketed_cell_count": 6,
|
|
"mean_absolute_error_req_s_per_gpu": 0.19548611111111105,
|
|
"mean_absolute_percentage_error": 0.07697140171082821,
|
|
"right_censored_cell_count": 6,
|
|
"root_mean_square_error_req_s_per_gpu": 0.2775267963371919
|
|
},
|
|
"statistics": {
|
|
"confidence_intervals": null,
|
|
"frontier_trials_per_anchor": 1,
|
|
"real_trials_per_anchor": 1,
|
|
"reason": "single retrospective observation per paired anchor"
|
|
},
|
|
"status": "VALID_RETROSPECTIVE_ALIGNED_ANALYSIS"
|
|
}
|