{ "comparison_scope": { "dash0_methods_vs_dash1_simulator": "indicative only: matched model/engine/workload/GPU type, different host and campaign", "pure_llm_vs_guided_harness": "direct: same dash0 fixed task context", "simulator_vs_surface": "direct: simulator predictions and exact dash1 12-cell real surface" }, "cost_definition": { "formula": "parallel_size * (last_engine_log_timestamp - first_engine_log_timestamp) / 3600", "future_all_in_metric": "allocation_start_to_GPU_idle * allocated_GPU_count, including failures", "included": [ "engine startup after first timestamp", "warm-up/probes until last timestamp" ], "not_reconstructable": [ "GPU allocation before first engine timestamp", "controller/LLM latency", "cleanup after last engine timestamp", "one-time simulator operator profiling GPU-hours" ], "reported_metric": "engine H20-hours lower bound" }, "data_sanity": { "invariants": { "all_costs_non_negative": true, "all_regrets_in_0_1": true, "dash0_task_contexts_equal_except_method_and_port": true, "engine_log_timestamps_monotonic": true, "sequential_scores_not_all_identical": true, "sequential_trial_counts_match_manifest": true, "simulator_scores_not_all_identical": true, "simulator_suite_has_no_failed_runs": true, "surface_has_expected_cell_count": true, "surface_matches_simulator_real_scores": true, "surface_scores_not_all_identical": true }, "regret_summary": { "distinct_n": 6, "max": 0.34328358208955223, "min": 0.0, "n": 16 }, "sequential_score_summary": { "distinct_n": 7, "max": 3.35, "min": 1.1041666666666667, "n": 9 }, "simulator_throughput_score_summary": { "distinct_n": 10, "max": 4.356763578770651, "min": 1.5449814460277083, "n": 12 }, "surface_score_summary": { "distinct_n": 8, "max": 3.283333333333333, "min": 1.2833333333333334, "n": 12 }, "trial_cost_summary": { "distinct_n": 26, "max": 0.49777777777777776, "min": 0.0, "n": 32 } }, "direct_dash0_comparison": { "regret_le_0.02": { "guided_harness_h20_hours_lower_bound": 0.4458333333333333, "guided_saving_vs_pure_llm": 0.6109090909090908, "pure_llm_h20_hours_lower_bound": 1.1458333333333333 }, "regret_le_0.05": { "guided_harness_h20_hours_lower_bound": 0.26805555555555555, "guided_saving_vs_pure_llm": 0.0585365853658536, "pure_llm_h20_hours_lower_bound": 0.2847222222222222 } }, "empirical_reference": { "meaning": "best observed across the two dash0 sequential runs; not a global oracle", "score_req_s_per_gpu": 3.35 }, "provisional_development_targets": { "five_percent_regret": { "development_target_h20_hours_lower_bound": 0.21444444444444444, "thirty_percent_below_posthoc_sim_slo_real_final": 0.36088888888888887, "twenty_percent_below_current_guided": 0.21444444444444444 }, "status": "lower-bound, single-task targets; require same-host prospective validation", "two_percent_regret": { "development_target_h20_hours_lower_bound": 0.3566666666666667, "thirty_percent_below_posthoc_sim_slo_real_final": 0.36088888888888887, "twenty_percent_below_current_guided": 0.3566666666666667 } }, "real_surface": { "cells": { "tp1_mns16": { "engine_h20_hours_lower_bound": 0.14083333333333334, "score_req_s_per_gpu": 2.35 }, "tp1_mns32": { "engine_h20_hours_lower_bound": 0.13194444444444445, "score_req_s_per_gpu": 2.283333333333333 }, "tp1_mns64": { "engine_h20_hours_lower_bound": 0.13527777777777777, "score_req_s_per_gpu": 2.283333333333333 }, "tp1_mns8": { "engine_h20_hours_lower_bound": 0.13333333333333333, "score_req_s_per_gpu": 2.1 }, "tp2_mns16": { "engine_h20_hours_lower_bound": 0.29944444444444446, "score_req_s_per_gpu": 2.275 }, "tp2_mns32": { "engine_h20_hours_lower_bound": 0.2816666666666667, "score_req_s_per_gpu": 3.283333333333333 }, "tp2_mns64": { "engine_h20_hours_lower_bound": 0.2338888888888889, "score_req_s_per_gpu": 3.2583333333333333 }, "tp2_mns8": { "engine_h20_hours_lower_bound": 0.2777777777777778, "score_req_s_per_gpu": 2.275 }, "tp4_mns16": { "engine_h20_hours_lower_bound": 0.4866666666666667, "score_req_s_per_gpu": 2.441666666666667 }, "tp4_mns32": { "engine_h20_hours_lower_bound": 0.49777777777777776, "score_req_s_per_gpu": 2.441666666666667 }, "tp4_mns64": { "engine_h20_hours_lower_bound": 0.49666666666666665, "score_req_s_per_gpu": 2.441666666666667 }, "tp4_mns8": { "engine_h20_hours_lower_bound": 0.48, "score_req_s_per_gpu": 1.2833333333333334 } }, "completed_annotation_engine_h20_hours_lower_bound": 3.5952777777777776, "failed_primary_attempt_engine_h20_hours_lower_bound": 0.0, "failed_primary_attempt_n": 4, "oracle_cells": [ "tp2_mns32" ], "oracle_score_req_s_per_gpu": 3.283333333333333, "primary_state_sha256": "790bf6df9045e22c46a8abdc022390bf842d884ef13ade4b627936722b9c3366", "tp4_companion_state_sha256": "1fc4f584a69a90fb0ff1d64f6a07a8383c368f4e306a0b7a3c19d2304042fd73" }, "schema": "aituner-tuning-cost-analysis-v1", "sequential_runs": { "guided_harness": { "curve": { "cost_to_threshold": { "regret_le_0": null, "regret_le_0.01": null, "regret_le_0.02": { "best_score_req_s_per_gpu": 3.283333333333333, "cumulative_engine_h20_hours_lower_bound": 0.4458333333333333, "regret": 0.01990049751243783, "trial_id": "trial-0003" }, "regret_le_0.05": { "best_score_req_s_per_gpu": 3.2583333333333333, "cumulative_engine_h20_hours_lower_bound": 0.26805555555555555, "regret": 0.027363184079602032, "trial_id": "trial-0002" } }, "points": [ { "best_score_req_s_per_gpu": 2.3833333333333333, "cumulative_engine_h20_hours_lower_bound": 0.09305555555555556, "regret": 0.2885572139303483, "trial_id": "trial-0001" }, { "best_score_req_s_per_gpu": 3.2583333333333333, "cumulative_engine_h20_hours_lower_bound": 0.26805555555555555, "regret": 0.027363184079602032, "trial_id": "trial-0002" }, { "best_score_req_s_per_gpu": 3.283333333333333, "cumulative_engine_h20_hours_lower_bound": 0.4458333333333333, "regret": 0.01990049751243783, "trial_id": "trial-0003" }, { "best_score_req_s_per_gpu": 3.3, "cumulative_engine_h20_hours_lower_bound": 0.6230555555555555, "regret": 0.014925373134328401, "trial_id": "trial-0004" } ], "reference_score_req_s_per_gpu": 3.35, "total_engine_h20_hours_lower_bound": 0.6230555555555555 }, "root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-harness-guided-v2", "state_sha256": "76e439d1f20e0f9af54785005258cbe39f205ae107fc2398955e360e263c7718", "study_spec_sha256": "03241089052f1a07dbbb2ba6100c6737635877b683d275813b3c59f2769fc2d7", "trials": [ { "config_patch": { "env_patch": {}, "flag_patch": {} }, "duration_note": "", "duration_s": 335.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.09305555555555556, "engine_log_sha256": "6d8b908085c03d4796e78679eed13737581fc7d2e18947fb72d7031a9f557c9d", "engine_timestamp_n": 108, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 1, "score_req_s_per_gpu": 2.3833333333333333, "status": "completed", "trial_id": "trial-0001" }, { "config_patch": { "env_patch": {}, "flag_patch": { "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 315.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.175, "engine_log_sha256": "b09caadd930495c27effea57e4164e52f58c6461cbbc0f4e873a309d45ec3443", "engine_timestamp_n": 134, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": 3.2583333333333333, "status": "completed", "trial_id": "trial-0002" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 16384, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 320.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.17777777777777778, "engine_log_sha256": "8f6f2637cff8566d39b8e6c17de71ed37a01f2aa6a4460ff36ecc9728d033c3a", "engine_timestamp_n": 134, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": 3.283333333333333, "status": "completed", "trial_id": "trial-0003" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 24576, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 319.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.17722222222222223, "engine_log_sha256": "29b8fc588b5422ceae60456188d6373070c179610243258fa213aeead9f07a00", "engine_timestamp_n": 137, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": 3.3, "status": "completed", "trial_id": "trial-0004" } ] }, "pure_llm": { "curve": { "cost_to_threshold": { "regret_le_0": { "best_score_req_s_per_gpu": 3.35, "cumulative_engine_h20_hours_lower_bound": 1.3719444444444444, "regret": 0.0, "trial_id": "trial-0007" }, "regret_le_0.01": { "best_score_req_s_per_gpu": 3.35, "cumulative_engine_h20_hours_lower_bound": 1.3719444444444444, "regret": 0.0, "trial_id": "trial-0007" }, "regret_le_0.02": { "best_score_req_s_per_gpu": 3.3, "cumulative_engine_h20_hours_lower_bound": 1.1458333333333333, "regret": 0.014925373134328401, "trial_id": "trial-0006" }, "regret_le_0.05": { "best_score_req_s_per_gpu": 3.2583333333333333, "cumulative_engine_h20_hours_lower_bound": 0.2847222222222222, "regret": 0.027363184079602032, "trial_id": "trial-0002" } }, "points": [ { "best_score_req_s_per_gpu": 2.2, "cumulative_engine_h20_hours_lower_bound": 0.10861111111111112, "regret": 0.34328358208955223, "trial_id": "trial-0001" }, { "best_score_req_s_per_gpu": 3.2583333333333333, "cumulative_engine_h20_hours_lower_bound": 0.2847222222222222, "regret": 0.027363184079602032, "trial_id": "trial-0002" }, { "best_score_req_s_per_gpu": 3.2583333333333333, "cumulative_engine_h20_hours_lower_bound": 0.29194444444444445, "regret": 0.027363184079602032, "trial_id": "trial-0003" }, { "best_score_req_s_per_gpu": 3.2583333333333333, "cumulative_engine_h20_hours_lower_bound": 0.4280555555555555, "regret": 0.027363184079602032, "trial_id": "trial-0004" }, { "best_score_req_s_per_gpu": 3.2583333333333333, "cumulative_engine_h20_hours_lower_bound": 0.9247222222222222, "regret": 0.027363184079602032, "trial_id": "trial-0005" }, { "best_score_req_s_per_gpu": 3.3, "cumulative_engine_h20_hours_lower_bound": 1.1458333333333333, "regret": 0.014925373134328401, "trial_id": "trial-0006" }, { "best_score_req_s_per_gpu": 3.35, "cumulative_engine_h20_hours_lower_bound": 1.3719444444444444, "regret": 0.0, "trial_id": "trial-0007" }, { "best_score_req_s_per_gpu": 3.35, "cumulative_engine_h20_hours_lower_bound": 1.561388888888889, "regret": 0.0, "trial_id": "trial-0008" }, { "best_score_req_s_per_gpu": 3.35, "cumulative_engine_h20_hours_lower_bound": 1.7497222222222222, "regret": 0.0, "trial_id": "trial-0009" }, { "best_score_req_s_per_gpu": 3.35, "cumulative_engine_h20_hours_lower_bound": 1.9047222222222222, "regret": 0.0, "trial_id": "trial-0010" }, { "best_score_req_s_per_gpu": 3.35, "cumulative_engine_h20_hours_lower_bound": 2.0930555555555554, "regret": 0.0, "trial_id": "trial-0011" }, { "best_score_req_s_per_gpu": 3.35, "cumulative_engine_h20_hours_lower_bound": 2.2824999999999998, "regret": 0.0, "trial_id": "trial-0012" } ], "reference_score_req_s_per_gpu": 3.35, "total_engine_h20_hours_lower_bound": 2.2824999999999998 }, "root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-noharness", "state_sha256": "342cde3546299cd312ebc078b1c9589773b841d23c4ed4632370883be598fd4a", "study_spec_sha256": "3238dc89477108405c09b25eee434c3c10c0f5be245d2bfa3b90509a372c088e", "trials": [ { "config_patch": { "env_patch": {}, "flag_patch": {} }, "duration_note": "", "duration_s": 391.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.10861111111111112, "engine_log_sha256": "c5caf9e9f6faa73a4a39adb71a7a6c04bcf623d5c0c91b231116bb596c8a11be", "engine_timestamp_n": 113, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 1, "score_req_s_per_gpu": 2.2, "status": "completed", "trial_id": "trial-0001" }, { "config_patch": { "env_patch": {}, "flag_patch": { "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 317.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.1761111111111111, "engine_log_sha256": "239ff7349d7b9a1b24334fbe6e2a45a967bfa2d16380e98da4f81520d422cd0d", "engine_timestamp_n": 134, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": 3.2583333333333333, "status": "completed", "trial_id": "trial-0002" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-expert-parallel": true, "expert-parallel-size": 2, "tensor-parallel-size": 2 } }, "duration_note": "conservative trial_spec-to-result mtime envelope for a pre-ready CLI failure", "duration_s": 13.0, "duration_source": "conservative_fallback", "engine_h20_hours_lower_bound": 0.007222222222222222, "engine_log_sha256": "4707735f33c8bf9030054eccd36b66985a3879050d80a853f93f166282414d8f", "engine_timestamp_n": 0, "engine_timestamps_monotonic": true, "failure_stage": "engine_launch", "parallel_size": 2, "score_req_s_per_gpu": null, "status": "failed", "trial_id": "trial-0003" }, { "config_patch": { "env_patch": {}, "flag_patch": { "data-parallel-size": 2, "tensor-parallel-size": 1 } }, "duration_note": "", "duration_s": 245.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.1361111111111111, "engine_log_sha256": "215ed31c6d227e4a6771ecb637c2b4e3e38e766d73a8882a923d1fa8b97a23da", "engine_timestamp_n": 170, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": null, "status": "completed", "trial_id": "trial-0004" }, { "config_patch": { "env_patch": {}, "flag_patch": { "data-parallel-size": 2, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 447.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.49666666666666665, "engine_log_sha256": "6057b3477ccca14e0ebd467ac96b777662df67b57827c851d58caf3a77b95ca4", "engine_timestamp_n": 201, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 4, "score_req_s_per_gpu": 1.1041666666666667, "status": "completed", "trial_id": "trial-0005" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 16384, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 398.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.22111111111111112, "engine_log_sha256": "c5c336f74d419645639b469eea9f92575780164637fdda078fa5c84c96915f1f", "engine_timestamp_n": 138, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": 3.3, "status": "completed", "trial_id": "trial-0006" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 24576, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 407.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.22611111111111112, "engine_log_sha256": "5e2829479ecd4771e7c781d40209f74e4c2843bbca15522974e435e06daa12a2", "engine_timestamp_n": 142, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": 3.35, "status": "completed", "trial_id": "trial-0007" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 32768, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 341.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.18944444444444444, "engine_log_sha256": "fde628968fba10fccfe5b09675aa57cf725f507be83c10dd3c0b6c109b5d72eb", "engine_timestamp_n": 136, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": null, "status": "completed", "trial_id": "trial-0008" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 28672, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 339.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.18833333333333332, "engine_log_sha256": "fc60765578d57af08e0310983ff42e2a200739a90f11c45f9d4b1f6b7b789cc8", "engine_timestamp_n": 136, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": null, "status": "completed", "trial_id": "trial-0009" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "enable-prefix-caching": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 24576, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 279.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.155, "engine_log_sha256": "22c473b39662df6c8c4c64a616346652ab889d63256efab18257b1f4445ce13b", "engine_timestamp_n": 133, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": null, "status": "completed", "trial_id": "trial-0010" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 25600, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 339.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.18833333333333332, "engine_log_sha256": "19b19de4a0a11a5a112b3e8ef04c3ca4d90d6c6c9097b5180c024d8570c9a608", "engine_timestamp_n": 136, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": null, "status": "completed", "trial_id": "trial-0011" }, { "config_patch": { "env_patch": {}, "flag_patch": { "enable-chunked-prefill": true, "gpu-memory-utilization": 0.95, "max-num-batched-tokens": 25088, "tensor-parallel-size": 2 } }, "duration_note": "", "duration_s": 341.0, "duration_source": "engine_log_span", "engine_h20_hours_lower_bound": 0.18944444444444444, "engine_log_sha256": "40f7307d6677f2bb55c0132d42fc82fce78ffa6a558d5c69b01ec60d1e40b3ae", "engine_timestamp_n": 136, "engine_timestamps_monotonic": true, "failure_stage": "", "parallel_size": 2, "score_req_s_per_gpu": null, "status": "completed", "trial_id": "trial-0012" } ] } }, "simulator": { "decision_bearing_throughput_proxy_plus_real_final": { "nominal_k_1": { "candidate_cells": [ "tp1_mns64" ], "engine_h20_hours_lower_bound": 0.13527777777777777, "real_evaluations": 1, "real_regret": 0.30456852791878175, "selected_cell": "tp1_mns64", "selected_real_score_req_s_per_gpu": 2.283333333333333 }, "nominal_k_2": { "candidate_cells": [ "tp1_mns64", "tp1_mns32" ], "engine_h20_hours_lower_bound": 0.26722222222222225, "real_evaluations": 2, "real_regret": 0.30456852791878175, "selected_cell": "tp1_mns64", "selected_real_score_req_s_per_gpu": 2.283333333333333 }, "nominal_k_3": { "candidate_cells": [ "tp1_mns64", "tp1_mns32", "tp2_mns32", "tp2_mns64" ], "engine_h20_hours_lower_bound": 0.7827777777777778, "real_evaluations": 4, "real_regret": 0.0, "selected_cell": "tp2_mns32", "selected_real_score_req_s_per_gpu": 3.283333333333333 } }, "decision_bearing_throughput_proxy_sim_only_top1": { "gpu_hours": 0.0, "real_regret": 0.30456852791878175, "selected_cell": "tp1_mns64", "selected_real_score_req_s_per_gpu": 2.283333333333333 }, "marginal_gpu_hours_without_real_verification": 0.0, "metrics_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206", "observed_fidelity_suite_cpu_hours": 2.055026211017717, "observed_fidelity_suite_runs": 184, "one_time_profile_cost_status": "not recorded; total cold-start cost is unknown", "one_time_profile_gpu_hours": null, "posthoc_slo_gated_plus_real_final": { "candidate_cells": [ "tp2_mns32", "tp2_mns64" ], "engine_h20_hours_lower_bound": 0.5155555555555555, "false_feasible": 21, "false_infeasible": 7, "real_evaluations": 2, "real_regret": 0.0, "selected_cell": "tp2_mns32", "selected_real_score_req_s_per_gpu": 3.283333333333333, "status": "diagnostic/post-hoc, not a preregistered prospective policy" } } }