gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Audit tuning cost and core challenges

Browse Source
This commit is contained in:
Gahow Wang
2026-07-15 01:41:25 +08:00
parent 8c930ba3a1
commit 0d16838097
5 changed files with 1733 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

507
runs/tuning-cost/analyze.py Normal file
View File

@@ -0,0 +1,507 @@
#!/usr/bin/env python3
"""Reconstruct tuning cost and cost-to-oracle curves from existing runs.
The historical engine logs do not contain controller setup/cleanup timestamps.
Consequently the reported GPU cost is an engine-lifetime lower bound:
parallel_size * (last engine timestamp - first engine timestamp). It must not
be presented as all-in method cost. New experiments should record allocation
start/end directly.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import re
import subprocess
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
SCHEMA = "aituner-tuning-cost-analysis-v1"
TIMESTAMP = re.compile(r"\b(\d{2}-\d{2} \d{2}:\d{2}:\d{2})(?:\.\d+)?\b")
def numeric_summary(values: list[float]) -> dict[str, Any]:
values = [float(value) for value in values]
return {
"n": len(values),
"min": min(values) if values else None,
"max": max(values) if values else None,
"distinct_n": len(set(values)),
}
def sha256_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
class Reader:
def __init__(self, repo_root: Path):
self.repo_root = repo_root
def _split(self, locator: str) -> tuple[str | None, str]:
if locator.startswith("ssh://"):
parsed = urlsplit(locator)
if not parsed.hostname or not parsed.path.startswith("/"):
raise ValueError(f"invalid SSH locator: {locator}")
return parsed.hostname, parsed.path
path = Path(locator)
if not path.is_absolute():
path = self.repo_root / path
return None, str(path)
def read_text(self, locator: str) -> str:
host, path = self._split(locator)
if host is None:
return Path(path).read_text(encoding="utf-8", errors="replace")
completed = subprocess.run(
["ssh", host, "cat", "--", path],
check=True,
capture_output=True,
text=True,
)
return completed.stdout
def join_locator(root: str, *parts: str) -> str:
return "/".join([root.rstrip("/"), *(part.strip("/") for part in parts)])
def timestamp_span(log_text: str, year: int) -> tuple[float | None, bool, int]:
parsed = [
datetime.strptime(f"{year}-{match}", "%Y-%m-%d %H:%M:%S")
for match in TIMESTAMP.findall(log_text)
]
if not parsed:
return None, True, 0
monotonic = all(right >= left for left, right in zip(parsed, parsed[1:]))
return (max(parsed) - min(parsed)).total_seconds(), monotonic, len(parsed)
def load_campaign(
reader: Reader,
root: str,
year: int,
missing_duration_s: dict[str, float] | None = None,
missing_reason: dict[str, str] | None = None,
) -> dict[str, Any]:
missing_duration_s = missing_duration_s or {}
missing_reason = missing_reason or {}
state_text = reader.read_text(join_locator(root, "state.json"))
state = json.loads(state_text)
trials = []
for trial in state["trials"]:
trial_id = str(trial["trial_id"])
log_text = reader.read_text(join_locator(root, "trials", trial_id, "engine.log"))
duration_s, monotonic, timestamp_n = timestamp_span(log_text, year)
duration_source = "engine_log_span"
duration_note = ""
if duration_s is None:
duration_s = float(missing_duration_s.get(trial_id, 0.0))
duration_source = (
"conservative_fallback" if trial_id in missing_duration_s else "no_engine_timestamp"
)
duration_note = missing_reason.get(trial_id, "")
parallel_size = int(trial["parallel_size"])
score = trial.get("best_request_rate_per_gpu")
trials.append(
{
"trial_id": trial_id,
"status": trial["status"],
"failure_stage": trial.get("failure_stage", ""),
"parallel_size": parallel_size,
"score_req_s_per_gpu": None if score is None else float(score),
"config_patch": trial["config_patch"],
"duration_s": duration_s,
"duration_source": duration_source,
"duration_note": duration_note,
"engine_timestamp_n": timestamp_n,
"engine_timestamps_monotonic": monotonic,
"engine_h20_hours_lower_bound": duration_s * parallel_size / 3600.0,
"engine_log_sha256": sha256_text(log_text),
}
)
return {
"root": root,
"state_sha256": sha256_text(state_text),
"trials": trials,
}
def fixed_task_context(spec: dict[str, Any]) -> dict[str, Any]:
flags = dict(spec["engine"]["base_flags"])
flags.pop("port", None)
return {
"model": spec["model"],
"hardware": spec["hardware"],
"engine_version": spec["engine"]["engine_version"],
"launch_args": spec["engine"]["launch_args"],
"base_flags_without_port": flags,
"search": spec["search"],
"slo": spec["slo"],
"trace": spec["trace"],
}
def regret(score: float, reference: float) -> float:
if reference <= 0:
raise ValueError("reference score must be positive")
return max(0.0, 1.0 - float(score) / float(reference))
def sequential_curve(
trials: list[dict[str, Any]], reference: float, thresholds: list[float]
) -> dict[str, Any]:
cumulative_cost = 0.0
best_score: float | None = None
points = []
for trial in trials:
cumulative_cost += float(trial["engine_h20_hours_lower_bound"])
score = trial["score_req_s_per_gpu"]
if score is not None:
best_score = score if best_score is None else max(best_score, score)
points.append(
{
"trial_id": trial["trial_id"],
"cumulative_engine_h20_hours_lower_bound": cumulative_cost,
"best_score_req_s_per_gpu": best_score,
"regret": None if best_score is None else regret(best_score, reference),
}
)
hits = {}
for threshold in thresholds:
hit = next(
(
point
for point in points
if point["regret"] is not None
and float(point["regret"]) <= float(threshold) + 1e-12
),
None,
)
hits[f"regret_le_{threshold:g}"] = hit
return {
"reference_score_req_s_per_gpu": reference,
"points": points,
"cost_to_threshold": hits,
"total_engine_h20_hours_lower_bound": cumulative_cost,
}
def surface_cell(trial: dict[str, Any]) -> str:
flags = trial["config_patch"]["flag_patch"]
return f"tp{int(flags['tensor-parallel-size'])}_mns{int(flags['max-num-seqs'])}"
def tie_expanded_candidates(scores: dict[str, float], nominal_k: int) -> list[str]:
ordered = sorted(scores, key=lambda cell: (-float(scores[cell]), cell))
if nominal_k <= 0 or nominal_k > len(ordered):
raise ValueError("nominal k outside score surface")
cutoff = float(scores[ordered[nominal_k - 1]])
tolerance = max(1e-12, abs(cutoff) * 1e-12)
return [cell for cell in ordered if float(scores[cell]) >= cutoff - tolerance]
def real_final_policy(
candidates: list[str], real_scores: dict[str, float], cell_costs: dict[str, float]
) -> dict[str, Any]:
oracle = max(real_scores.values())
selected = max(candidates, key=lambda cell: (real_scores[cell], cell))
return {
"candidate_cells": candidates,
"real_evaluations": len(candidates),
"selected_cell": selected,
"selected_real_score_req_s_per_gpu": real_scores[selected],
"real_regret": regret(real_scores[selected], oracle),
"engine_h20_hours_lower_bound": sum(cell_costs[cell] for cell in candidates),
}
def percentage_saving(new: float, old: float) -> float:
if old <= 0:
raise ValueError("baseline cost must be positive")
return 1.0 - new / old
def build_analysis(manifest_path: Path) -> dict[str, Any]:
repo_root = manifest_path.resolve().parents[2]
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
reader = Reader(repo_root)
year = int(manifest["year"])
sequential = {}
task_contexts = {}
for name, run in manifest["sequential_runs"].items():
sequential[name] = load_campaign(
reader,
run["root"],
year,
run.get("missing_log_duration_s"),
run.get("missing_log_reason"),
)
spec_text = reader.read_text(join_locator(run["root"], "study_spec.snapshot.json"))
sequential[name]["study_spec_sha256"] = sha256_text(spec_text)
task_contexts[name] = fixed_task_context(json.loads(spec_text))
all_sequential_scores = [
trial["score_req_s_per_gpu"]
for campaign in sequential.values()
for trial in campaign["trials"]
if trial["score_req_s_per_gpu"] is not None
]
empirical_reference = max(all_sequential_scores)
thresholds = [float(value) for value in manifest["threshold_regrets"]]
for campaign in sequential.values():
campaign["curve"] = sequential_curve(
campaign["trials"], empirical_reference, thresholds
)
surface_manifest = manifest["real_surface"]
primary = load_campaign(reader, surface_manifest["primary_root"], year)
companion = load_campaign(reader, surface_manifest["tp4_companion_root"], year)
completed_surface_trials = [
trial
for trial in primary["trials"] + companion["trials"]
if trial["status"] == "completed" and trial["score_req_s_per_gpu"] is not None
]
cells: dict[str, dict[str, Any]] = {}
for trial in completed_surface_trials:
flags = trial["config_patch"]["flag_patch"]
if int(flags["max-num-batched-tokens"]) != int(
surface_manifest["fixed_max_num_batched_tokens"]
):
raise ValueError("surface MBT invariant failed")
cell = surface_cell(trial)
if cell in cells:
raise ValueError(f"duplicate completed surface cell: {cell}")
cells[cell] = trial
real_scores = {cell: float(trial["score_req_s_per_gpu"]) for cell, trial in cells.items()}
cell_costs = {
cell: float(trial["engine_h20_hours_lower_bound"]) for cell, trial in cells.items()
}
surface_oracle_score = max(real_scores.values())
surface_oracle_cells = [
cell for cell, score in real_scores.items() if math.isclose(score, surface_oracle_score)
]
simulator_text = reader.read_text(manifest["simulator_metrics"])
simulator = json.loads(simulator_text)
simulator_real_scores = {
cell: float(score) for cell, score in simulator["real_scores"].items()
}
surface_matches_simulator = set(real_scores) == set(simulator_real_scores) and all(
math.isclose(real_scores[cell], simulator_real_scores[cell], abs_tol=1e-12)
for cell in real_scores
)
throughput = simulator["analyses"]["frozen-calibrated/throughput-proxy"]
throughput_scores = {
cell: float(score) for cell, score in throughput["simulated_scores"].items()
}
throughput_real_final = {
f"nominal_k_{k}": real_final_policy(
tie_expanded_candidates(throughput_scores, k), real_scores, cell_costs
)
for k in (1, 2, 3)
}
throughput_top1 = tie_expanded_candidates(throughput_scores, 1)
simulator_only_cell = throughput_top1[0]
slo = simulator["analyses"]["frozen-calibrated/SLO-gated"]
slo_top_bucket = list(slo["metrics"]["top1"]["candidate_cells"])
slo_diagnostic = real_final_policy(slo_top_bucket, real_scores, cell_costs)
pure_hits = sequential["pure_llm"]["curve"]["cost_to_threshold"]
guided_hits = sequential["guided_harness"]["curve"]["cost_to_threshold"]
direct_comparison = {}
for threshold in (0.05, 0.02):
key = f"regret_le_{threshold:g}"
pure_hit = pure_hits[key]
guided_hit = guided_hits[key]
if pure_hit is None or guided_hit is None:
continue
pure_cost = float(pure_hit["cumulative_engine_h20_hours_lower_bound"])
guided_cost = float(guided_hit["cumulative_engine_h20_hours_lower_bound"])
direct_comparison[key] = {
"pure_llm_h20_hours_lower_bound": pure_cost,
"guided_harness_h20_hours_lower_bound": guided_cost,
"guided_saving_vs_pure_llm": percentage_saving(guided_cost, pure_cost),
}
five_cost = direct_comparison["regret_le_0.05"][
"guided_harness_h20_hours_lower_bound"
]
two_cost = direct_comparison["regret_le_0.02"][
"guided_harness_h20_hours_lower_bound"
]
sim_real_cost = float(slo_diagnostic["engine_h20_hours_lower_bound"])
target_bars = {
"five_percent_regret": {
"twenty_percent_below_current_guided": 0.8 * five_cost,
"thirty_percent_below_posthoc_sim_slo_real_final": 0.7 * sim_real_cost,
"development_target_h20_hours_lower_bound": min(
0.8 * five_cost, 0.7 * sim_real_cost
),
},
"two_percent_regret": {
"twenty_percent_below_current_guided": 0.8 * two_cost,
"thirty_percent_below_posthoc_sim_slo_real_final": 0.7 * sim_real_cost,
"development_target_h20_hours_lower_bound": min(
0.8 * two_cost, 0.7 * sim_real_cost
),
},
}
all_trial_costs = [
float(trial["engine_h20_hours_lower_bound"])
for campaign in sequential.values()
for trial in campaign["trials"]
] + [
float(trial["engine_h20_hours_lower_bound"])
for campaign in (primary, companion)
for trial in campaign["trials"]
]
all_regrets = [
float(point["regret"])
for campaign in sequential.values()
for point in campaign["curve"]["points"]
if point["regret"] is not None
]
monotonic_logs = all(
trial["engine_timestamps_monotonic"]
for campaign in [*sequential.values(), primary, companion]
for trial in campaign["trials"]
)
invariants = {
"dash0_task_contexts_equal_except_method_and_port": len(
{json.dumps(value, sort_keys=True) for value in task_contexts.values()}
)
== 1,
"surface_has_expected_cell_count": len(cells)
== int(surface_manifest["expected_cells"]),
"surface_matches_simulator_real_scores": surface_matches_simulator,
"all_costs_non_negative": all(value >= 0 for value in all_trial_costs),
"all_regrets_in_0_1": all(0 <= value <= 1 for value in all_regrets),
"surface_scores_not_all_identical": len(set(real_scores.values())) > 1,
"sequential_scores_not_all_identical": len(set(all_sequential_scores)) > 1,
"engine_log_timestamps_monotonic": monotonic_logs,
"sequential_trial_counts_match_manifest": all(
len(sequential[name]["trials"]) == int(run["expected_trials"])
for name, run in manifest["sequential_runs"].items()
),
"simulator_suite_has_no_failed_runs": int(simulator["execution"]["failed_runs"])
== 0,
"simulator_scores_not_all_identical": len(set(throughput_scores.values())) > 1,
}
failed_invariants = [name for name, passed in invariants.items() if not passed]
if failed_invariants:
raise RuntimeError(f"data sanity invariant failed: {failed_invariants}")
failed_primary_attempts = [
trial for trial in primary["trials"] if trial["status"] != "completed"
]
return {
"schema": SCHEMA,
"cost_definition": {
"reported_metric": "engine H20-hours lower bound",
"formula": "parallel_size * (last_engine_log_timestamp - first_engine_log_timestamp) / 3600",
"included": ["engine startup after first timestamp", "warm-up/probes until last timestamp"],
"not_reconstructable": [
"GPU allocation before first engine timestamp",
"controller/LLM latency",
"cleanup after last engine timestamp",
"one-time simulator operator profiling GPU-hours",
],
"future_all_in_metric": "allocation_start_to_GPU_idle * allocated_GPU_count, including failures",
},
"comparison_scope": {
"pure_llm_vs_guided_harness": "direct: same dash0 fixed task context",
"simulator_vs_surface": "direct: simulator predictions and exact dash1 12-cell real surface",
"dash0_methods_vs_dash1_simulator": "indicative only: matched model/engine/workload/GPU type, different host and campaign",
},
"empirical_reference": {
"score_req_s_per_gpu": empirical_reference,
"meaning": "best observed across the two dash0 sequential runs; not a global oracle",
},
"sequential_runs": sequential,
"direct_dash0_comparison": direct_comparison,
"real_surface": {
"cells": {
cell: {
"score_req_s_per_gpu": real_scores[cell],
"engine_h20_hours_lower_bound": cell_costs[cell],
}
for cell in sorted(cells)
},
"oracle_score_req_s_per_gpu": surface_oracle_score,
"oracle_cells": surface_oracle_cells,
"completed_annotation_engine_h20_hours_lower_bound": sum(cell_costs.values()),
"failed_primary_attempt_n": len(failed_primary_attempts),
"failed_primary_attempt_engine_h20_hours_lower_bound": sum(
float(trial["engine_h20_hours_lower_bound"])
for trial in failed_primary_attempts
),
"primary_state_sha256": primary["state_sha256"],
"tp4_companion_state_sha256": companion["state_sha256"],
},
"simulator": {
"marginal_gpu_hours_without_real_verification": 0.0,
"observed_fidelity_suite_cpu_hours": float(
simulator["execution"]["suite_elapsed_seconds"]
)
/ 3600.0,
"observed_fidelity_suite_runs": int(simulator["execution"]["attempted_runs"]),
"one_time_profile_gpu_hours": None,
"one_time_profile_cost_status": "not recorded; total cold-start cost is unknown",
"decision_bearing_throughput_proxy_sim_only_top1": {
"selected_cell": simulator_only_cell,
"selected_real_score_req_s_per_gpu": real_scores[simulator_only_cell],
"real_regret": regret(real_scores[simulator_only_cell], surface_oracle_score),
"gpu_hours": 0.0,
},
"decision_bearing_throughput_proxy_plus_real_final": throughput_real_final,
"posthoc_slo_gated_plus_real_final": {
**slo_diagnostic,
"status": "diagnostic/post-hoc, not a preregistered prospective policy",
"false_feasible": int(slo["false_feasibility"]["overall"]["false_feasible"]),
"false_infeasible": int(slo["false_feasibility"]["overall"]["false_infeasible"]),
},
"metrics_sha256": sha256_text(simulator_text),
},
"provisional_development_targets": {
"status": "lower-bound, single-task targets; require same-host prospective validation",
**target_bars,
},
"data_sanity": {
"invariants": invariants,
"sequential_score_summary": numeric_summary(all_sequential_scores),
"surface_score_summary": numeric_summary(list(real_scores.values())),
"simulator_throughput_score_summary": numeric_summary(
list(throughput_scores.values())
),
"trial_cost_summary": numeric_summary(all_trial_costs),
"regret_summary": numeric_summary(all_regrets),
},
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--manifest", type=Path, default=Path(__file__).with_name("manifest.json"))
parser.add_argument("--output", type=Path, default=Path(__file__).with_name("metrics.json"))
args = parser.parse_args()
analysis = build_analysis(args.manifest)
args.output.write_text(json.dumps(analysis, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps({
"status": "ok",
"output": str(args.output),
"empirical_reference": analysis["empirical_reference"],
"surface_oracle": analysis["real_surface"]["oracle_score_req_s_per_gpu"],
"sanity": analysis["data_sanity"],
}, indent=2, sort_keys=True))
if __name__ == "__main__":
main()

42
runs/tuning-cost/manifest.json Normal file
View File

@@ -0,0 +1,42 @@
{
"schema": "aituner-tuning-cost-v1",
"year": 2026,
"task": {
"model": "Qwen/Qwen3-30B-A3B",
"engine": "community-vLLM 0.20.0",
"gpu": "NVIDIA H20",
"trace_window": "chat_w20260311_1000",
"input_tokens": [0, 8192],
"output_tokens": 128,
"replay_time_scale": 0.1,
"target_pass_rate": 0.95,
"ttft_ms": [2000, 4000, 6000],
"tpot_ms": 50
},
"sequential_runs": {
"pure_llm": {
"root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-noharness",
"expected_trials": 12,
"missing_log_duration_s": {
"trial-0003": 13.0
},
"missing_log_reason": {
"trial-0003": "conservative trial_spec-to-result mtime envelope for a pre-ready CLI failure"
}
},
"guided_harness": {
"root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-harness-guided-v2",
"expected_trials": 4,
"missing_log_duration_s": {},
"missing_log_reason": {}
}
},
"real_surface": {
"primary_root": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z/store/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z",
"tp4_companion_root": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp4-mns-nocap-qps20-dash1-d8899c5-20260701T161900Z/store/interaction-mixed-qwen30b-tp4-mns-nocap-qps20-dash1-d8899c5-20260701T161900Z",
"expected_cells": 12,
"fixed_max_num_batched_tokens": 8192
},
"simulator_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json",
"threshold_regrets": [0.05, 0.02, 0.01, 0.0]
}

736
runs/tuning-cost/metrics.json Normal file
View File

@@ -0,0 +1,736 @@
{
"comparison_scope": {
"dash0_methods_vs_dash1_simulator": "indicative only: matched model/engine/workload/GPU type, different host and campaign",
"pure_llm_vs_guided_harness": "direct: same dash0 fixed task context",
"simulator_vs_surface": "direct: simulator predictions and exact dash1 12-cell real surface"
},
"cost_definition": {
"formula": "parallel_size * (last_engine_log_timestamp - first_engine_log_timestamp) / 3600",
"future_all_in_metric": "allocation_start_to_GPU_idle * allocated_GPU_count, including failures",
"included": [
"engine startup after first timestamp",
"warm-up/probes until last timestamp"
],
"not_reconstructable": [
"GPU allocation before first engine timestamp",
"controller/LLM latency",
"cleanup after last engine timestamp",
"one-time simulator operator profiling GPU-hours"
],
"reported_metric": "engine H20-hours lower bound"
},
"data_sanity": {
"invariants": {
"all_costs_non_negative": true,
"all_regrets_in_0_1": true,
"dash0_task_contexts_equal_except_method_and_port": true,
"engine_log_timestamps_monotonic": true,
"sequential_scores_not_all_identical": true,
"sequential_trial_counts_match_manifest": true,
"simulator_scores_not_all_identical": true,
"simulator_suite_has_no_failed_runs": true,
"surface_has_expected_cell_count": true,
"surface_matches_simulator_real_scores": true,
"surface_scores_not_all_identical": true
},
"regret_summary": {
"distinct_n": 6,
"max": 0.34328358208955223,
"min": 0.0,
"n": 16
},
"sequential_score_summary": {
"distinct_n": 7,
"max": 3.35,
"min": 1.1041666666666667,
"n": 9
},
"simulator_throughput_score_summary": {
"distinct_n": 10,
"max": 4.356763578770651,
"min": 1.5449814460277083,
"n": 12
},
"surface_score_summary": {
"distinct_n": 8,
"max": 3.283333333333333,
"min": 1.2833333333333334,
"n": 12
},
"trial_cost_summary": {
"distinct_n": 26,
"max": 0.49777777777777776,
"min": 0.0,
"n": 32
}
},
"direct_dash0_comparison": {
"regret_le_0.02": {
"guided_harness_h20_hours_lower_bound": 0.4458333333333333,
"guided_saving_vs_pure_llm": 0.6109090909090908,
"pure_llm_h20_hours_lower_bound": 1.1458333333333333
},
"regret_le_0.05": {
"guided_harness_h20_hours_lower_bound": 0.26805555555555555,
"guided_saving_vs_pure_llm": 0.0585365853658536,
"pure_llm_h20_hours_lower_bound": 0.2847222222222222
}
},
"empirical_reference": {
"meaning": "best observed across the two dash0 sequential runs; not a global oracle",
"score_req_s_per_gpu": 3.35
},
"provisional_development_targets": {
"five_percent_regret": {
"development_target_h20_hours_lower_bound": 0.21444444444444444,
"thirty_percent_below_posthoc_sim_slo_real_final": 0.36088888888888887,
"twenty_percent_below_current_guided": 0.21444444444444444
},
"status": "lower-bound, single-task targets; require same-host prospective validation",
"two_percent_regret": {
"development_target_h20_hours_lower_bound": 0.3566666666666667,
"thirty_percent_below_posthoc_sim_slo_real_final": 0.36088888888888887,
"twenty_percent_below_current_guided": 0.3566666666666667
}
},
"real_surface": {
"cells": {
"tp1_mns16": {
"engine_h20_hours_lower_bound": 0.14083333333333334,
"score_req_s_per_gpu": 2.35
},
"tp1_mns32": {
"engine_h20_hours_lower_bound": 0.13194444444444445,
"score_req_s_per_gpu": 2.283333333333333
},
"tp1_mns64": {
"engine_h20_hours_lower_bound": 0.13527777777777777,
"score_req_s_per_gpu": 2.283333333333333
},
"tp1_mns8": {
"engine_h20_hours_lower_bound": 0.13333333333333333,
"score_req_s_per_gpu": 2.1
},
"tp2_mns16": {
"engine_h20_hours_lower_bound": 0.29944444444444446,
"score_req_s_per_gpu": 2.275
},
"tp2_mns32": {
"engine_h20_hours_lower_bound": 0.2816666666666667,
"score_req_s_per_gpu": 3.283333333333333
},
"tp2_mns64": {
"engine_h20_hours_lower_bound": 0.2338888888888889,
"score_req_s_per_gpu": 3.2583333333333333
},
"tp2_mns8": {
"engine_h20_hours_lower_bound": 0.2777777777777778,
"score_req_s_per_gpu": 2.275
},
"tp4_mns16": {
"engine_h20_hours_lower_bound": 0.4866666666666667,
"score_req_s_per_gpu": 2.441666666666667
},
"tp4_mns32": {
"engine_h20_hours_lower_bound": 0.49777777777777776,
"score_req_s_per_gpu": 2.441666666666667
},
"tp4_mns64": {
"engine_h20_hours_lower_bound": 0.49666666666666665,
"score_req_s_per_gpu": 2.441666666666667
},
"tp4_mns8": {
"engine_h20_hours_lower_bound": 0.48,
"score_req_s_per_gpu": 1.2833333333333334
}
},
"completed_annotation_engine_h20_hours_lower_bound": 3.5952777777777776,
"failed_primary_attempt_engine_h20_hours_lower_bound": 0.0,
"failed_primary_attempt_n": 4,
"oracle_cells": [
"tp2_mns32"
],
"oracle_score_req_s_per_gpu": 3.283333333333333,
"primary_state_sha256": "790bf6df9045e22c46a8abdc022390bf842d884ef13ade4b627936722b9c3366",
"tp4_companion_state_sha256": "1fc4f584a69a90fb0ff1d64f6a07a8383c368f4e306a0b7a3c19d2304042fd73"
},
"schema": "aituner-tuning-cost-analysis-v1",
"sequential_runs": {
"guided_harness": {
"curve": {
"cost_to_threshold": {
"regret_le_0": null,
"regret_le_0.01": null,
"regret_le_0.02": {
"best_score_req_s_per_gpu": 3.283333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.4458333333333333,
"regret": 0.01990049751243783,
"trial_id": "trial-0003"
},
"regret_le_0.05": {
"best_score_req_s_per_gpu": 3.2583333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.26805555555555555,
"regret": 0.027363184079602032,
"trial_id": "trial-0002"
}
},
"points": [
{
"best_score_req_s_per_gpu": 2.3833333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.09305555555555556,
"regret": 0.2885572139303483,
"trial_id": "trial-0001"
},
{
"best_score_req_s_per_gpu": 3.2583333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.26805555555555555,
"regret": 0.027363184079602032,
"trial_id": "trial-0002"
},
{
"best_score_req_s_per_gpu": 3.283333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.4458333333333333,
"regret": 0.01990049751243783,
"trial_id": "trial-0003"
},
{
"best_score_req_s_per_gpu": 3.3,
"cumulative_engine_h20_hours_lower_bound": 0.6230555555555555,
"regret": 0.014925373134328401,
"trial_id": "trial-0004"
}
],
"reference_score_req_s_per_gpu": 3.35,
"total_engine_h20_hours_lower_bound": 0.6230555555555555
},
"root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-harness-guided-v2",
"state_sha256": "76e439d1f20e0f9af54785005258cbe39f205ae107fc2398955e360e263c7718",
"study_spec_sha256": "03241089052f1a07dbbb2ba6100c6737635877b683d275813b3c59f2769fc2d7",
"trials": [
{
"config_patch": {
"env_patch": {},
"flag_patch": {}
},
"duration_note": "",
"duration_s": 335.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.09305555555555556,
"engine_log_sha256": "6d8b908085c03d4796e78679eed13737581fc7d2e18947fb72d7031a9f557c9d",
"engine_timestamp_n": 108,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 1,
"score_req_s_per_gpu": 2.3833333333333333,
"status": "completed",
"trial_id": "trial-0001"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 315.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.175,
"engine_log_sha256": "b09caadd930495c27effea57e4164e52f58c6461cbbc0f4e873a309d45ec3443",
"engine_timestamp_n": 134,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": 3.2583333333333333,
"status": "completed",
"trial_id": "trial-0002"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 16384,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 320.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.17777777777777778,
"engine_log_sha256": "8f6f2637cff8566d39b8e6c17de71ed37a01f2aa6a4460ff36ecc9728d033c3a",
"engine_timestamp_n": 134,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": 3.283333333333333,
"status": "completed",
"trial_id": "trial-0003"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 24576,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 319.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.17722222222222223,
"engine_log_sha256": "29b8fc588b5422ceae60456188d6373070c179610243258fa213aeead9f07a00",
"engine_timestamp_n": 137,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": 3.3,
"status": "completed",
"trial_id": "trial-0004"
}
]
},
"pure_llm": {
"curve": {
"cost_to_threshold": {
"regret_le_0": {
"best_score_req_s_per_gpu": 3.35,
"cumulative_engine_h20_hours_lower_bound": 1.3719444444444444,
"regret": 0.0,
"trial_id": "trial-0007"
},
"regret_le_0.01": {
"best_score_req_s_per_gpu": 3.35,
"cumulative_engine_h20_hours_lower_bound": 1.3719444444444444,
"regret": 0.0,
"trial_id": "trial-0007"
},
"regret_le_0.02": {
"best_score_req_s_per_gpu": 3.3,
"cumulative_engine_h20_hours_lower_bound": 1.1458333333333333,
"regret": 0.014925373134328401,
"trial_id": "trial-0006"
},
"regret_le_0.05": {
"best_score_req_s_per_gpu": 3.2583333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.2847222222222222,
"regret": 0.027363184079602032,
"trial_id": "trial-0002"
}
},
"points": [
{
"best_score_req_s_per_gpu": 2.2,
"cumulative_engine_h20_hours_lower_bound": 0.10861111111111112,
"regret": 0.34328358208955223,
"trial_id": "trial-0001"
},
{
"best_score_req_s_per_gpu": 3.2583333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.2847222222222222,
"regret": 0.027363184079602032,
"trial_id": "trial-0002"
},
{
"best_score_req_s_per_gpu": 3.2583333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.29194444444444445,
"regret": 0.027363184079602032,
"trial_id": "trial-0003"
},
{
"best_score_req_s_per_gpu": 3.2583333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.4280555555555555,
"regret": 0.027363184079602032,
"trial_id": "trial-0004"
},
{
"best_score_req_s_per_gpu": 3.2583333333333333,
"cumulative_engine_h20_hours_lower_bound": 0.9247222222222222,
"regret": 0.027363184079602032,
"trial_id": "trial-0005"
},
{
"best_score_req_s_per_gpu": 3.3,
"cumulative_engine_h20_hours_lower_bound": 1.1458333333333333,
"regret": 0.014925373134328401,
"trial_id": "trial-0006"
},
{
"best_score_req_s_per_gpu": 3.35,
"cumulative_engine_h20_hours_lower_bound": 1.3719444444444444,
"regret": 0.0,
"trial_id": "trial-0007"
},
{
"best_score_req_s_per_gpu": 3.35,
"cumulative_engine_h20_hours_lower_bound": 1.561388888888889,
"regret": 0.0,
"trial_id": "trial-0008"
},
{
"best_score_req_s_per_gpu": 3.35,
"cumulative_engine_h20_hours_lower_bound": 1.7497222222222222,
"regret": 0.0,
"trial_id": "trial-0009"
},
{
"best_score_req_s_per_gpu": 3.35,
"cumulative_engine_h20_hours_lower_bound": 1.9047222222222222,
"regret": 0.0,
"trial_id": "trial-0010"
},
{
"best_score_req_s_per_gpu": 3.35,
"cumulative_engine_h20_hours_lower_bound": 2.0930555555555554,
"regret": 0.0,
"trial_id": "trial-0011"
},
{
"best_score_req_s_per_gpu": 3.35,
"cumulative_engine_h20_hours_lower_bound": 2.2824999999999998,
"regret": 0.0,
"trial_id": "trial-0012"
}
],
"reference_score_req_s_per_gpu": 3.35,
"total_engine_h20_hours_lower_bound": 2.2824999999999998
},
"root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-noharness",
"state_sha256": "342cde3546299cd312ebc078b1c9589773b841d23c4ed4632370883be598fd4a",
"study_spec_sha256": "3238dc89477108405c09b25eee434c3c10c0f5be245d2bfa3b90509a372c088e",
"trials": [
{
"config_patch": {
"env_patch": {},
"flag_patch": {}
},
"duration_note": "",
"duration_s": 391.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.10861111111111112,
"engine_log_sha256": "c5caf9e9f6faa73a4a39adb71a7a6c04bcf623d5c0c91b231116bb596c8a11be",
"engine_timestamp_n": 113,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 1,
"score_req_s_per_gpu": 2.2,
"status": "completed",
"trial_id": "trial-0001"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 317.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.1761111111111111,
"engine_log_sha256": "239ff7349d7b9a1b24334fbe6e2a45a967bfa2d16380e98da4f81520d422cd0d",
"engine_timestamp_n": 134,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": 3.2583333333333333,
"status": "completed",
"trial_id": "trial-0002"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-expert-parallel": true,
"expert-parallel-size": 2,
"tensor-parallel-size": 2
}
},
"duration_note": "conservative trial_spec-to-result mtime envelope for a pre-ready CLI failure",
"duration_s": 13.0,
"duration_source": "conservative_fallback",
"engine_h20_hours_lower_bound": 0.007222222222222222,
"engine_log_sha256": "4707735f33c8bf9030054eccd36b66985a3879050d80a853f93f166282414d8f",
"engine_timestamp_n": 0,
"engine_timestamps_monotonic": true,
"failure_stage": "engine_launch",
"parallel_size": 2,
"score_req_s_per_gpu": null,
"status": "failed",
"trial_id": "trial-0003"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"data-parallel-size": 2,
"tensor-parallel-size": 1
}
},
"duration_note": "",
"duration_s": 245.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.1361111111111111,
"engine_log_sha256": "215ed31c6d227e4a6771ecb637c2b4e3e38e766d73a8882a923d1fa8b97a23da",
"engine_timestamp_n": 170,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": null,
"status": "completed",
"trial_id": "trial-0004"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"data-parallel-size": 2,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 447.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.49666666666666665,
"engine_log_sha256": "6057b3477ccca14e0ebd467ac96b777662df67b57827c851d58caf3a77b95ca4",
"engine_timestamp_n": 201,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 4,
"score_req_s_per_gpu": 1.1041666666666667,
"status": "completed",
"trial_id": "trial-0005"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 16384,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 398.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.22111111111111112,
"engine_log_sha256": "c5c336f74d419645639b469eea9f92575780164637fdda078fa5c84c96915f1f",
"engine_timestamp_n": 138,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": 3.3,
"status": "completed",
"trial_id": "trial-0006"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 24576,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 407.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.22611111111111112,
"engine_log_sha256": "5e2829479ecd4771e7c781d40209f74e4c2843bbca15522974e435e06daa12a2",
"engine_timestamp_n": 142,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": 3.35,
"status": "completed",
"trial_id": "trial-0007"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 32768,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 341.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.18944444444444444,
"engine_log_sha256": "fde628968fba10fccfe5b09675aa57cf725f507be83c10dd3c0b6c109b5d72eb",
"engine_timestamp_n": 136,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": null,
"status": "completed",
"trial_id": "trial-0008"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 28672,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 339.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.18833333333333332,
"engine_log_sha256": "fc60765578d57af08e0310983ff42e2a200739a90f11c45f9d4b1f6b7b789cc8",
"engine_timestamp_n": 136,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": null,
"status": "completed",
"trial_id": "trial-0009"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"enable-prefix-caching": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 24576,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 279.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.155,
"engine_log_sha256": "22c473b39662df6c8c4c64a616346652ab889d63256efab18257b1f4445ce13b",
"engine_timestamp_n": 133,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": null,
"status": "completed",
"trial_id": "trial-0010"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 25600,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 339.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.18833333333333332,
"engine_log_sha256": "19b19de4a0a11a5a112b3e8ef04c3ca4d90d6c6c9097b5180c024d8570c9a608",
"engine_timestamp_n": 136,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": null,
"status": "completed",
"trial_id": "trial-0011"
},
{
"config_patch": {
"env_patch": {},
"flag_patch": {
"enable-chunked-prefill": true,
"gpu-memory-utilization": 0.95,
"max-num-batched-tokens": 25088,
"tensor-parallel-size": 2
}
},
"duration_note": "",
"duration_s": 341.0,
"duration_source": "engine_log_span",
"engine_h20_hours_lower_bound": 0.18944444444444444,
"engine_log_sha256": "40f7307d6677f2bb55c0132d42fc82fce78ffa6a558d5c69b01ec60d1e40b3ae",
"engine_timestamp_n": 136,
"engine_timestamps_monotonic": true,
"failure_stage": "",
"parallel_size": 2,
"score_req_s_per_gpu": null,
"status": "completed",
"trial_id": "trial-0012"
}
]
}
},
"simulator": {
"decision_bearing_throughput_proxy_plus_real_final": {
"nominal_k_1": {
"candidate_cells": [
"tp1_mns64"
],
"engine_h20_hours_lower_bound": 0.13527777777777777,
"real_evaluations": 1,
"real_regret": 0.30456852791878175,
"selected_cell": "tp1_mns64",
"selected_real_score_req_s_per_gpu": 2.283333333333333
},
"nominal_k_2": {
"candidate_cells": [
"tp1_mns64",
"tp1_mns32"
],
"engine_h20_hours_lower_bound": 0.26722222222222225,
"real_evaluations": 2,
"real_regret": 0.30456852791878175,
"selected_cell": "tp1_mns64",
"selected_real_score_req_s_per_gpu": 2.283333333333333
},
"nominal_k_3": {
"candidate_cells": [
"tp1_mns64",
"tp1_mns32",
"tp2_mns32",
"tp2_mns64"
],
"engine_h20_hours_lower_bound": 0.7827777777777778,
"real_evaluations": 4,
"real_regret": 0.0,
"selected_cell": "tp2_mns32",
"selected_real_score_req_s_per_gpu": 3.283333333333333
}
},
"decision_bearing_throughput_proxy_sim_only_top1": {
"gpu_hours": 0.0,
"real_regret": 0.30456852791878175,
"selected_cell": "tp1_mns64",
"selected_real_score_req_s_per_gpu": 2.283333333333333
},
"marginal_gpu_hours_without_real_verification": 0.0,
"metrics_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206",
"observed_fidelity_suite_cpu_hours": 2.055026211017717,
"observed_fidelity_suite_runs": 184,
"one_time_profile_cost_status": "not recorded; total cold-start cost is unknown",
"one_time_profile_gpu_hours": null,
"posthoc_slo_gated_plus_real_final": {
"candidate_cells": [
"tp2_mns32",
"tp2_mns64"
],
"engine_h20_hours_lower_bound": 0.5155555555555555,
"false_feasible": 21,
"false_infeasible": 7,
"real_evaluations": 2,
"real_regret": 0.0,
"selected_cell": "tp2_mns32",
"selected_real_score_req_s_per_gpu": 3.283333333333333,
"status": "diagnostic/post-hoc, not a preregistered prospective policy"
}
}
}

57
runs/tuning-cost/test_analysis.py Normal file
View File

@@ -0,0 +1,57 @@
#!/usr/bin/env python3
from __future__ import annotations
import importlib.util
import math
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
def load_analysis():
spec = importlib.util.spec_from_file_location("tuning_cost", HERE / "analyze.py")
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def main() -> None:
analysis = load_analysis()
duration, monotonic, count = analysis.timestamp_span(
"INFO 07-01 10:00:00 x\nINFO 07-01 10:05:30 y\n", 2026
)
assert duration == 330.0
assert monotonic
assert count == 2
trials = [
{"trial_id": "t1", "engine_h20_hours_lower_bound": 0.1, "score_req_s_per_gpu": 8.0},
{"trial_id": "t2", "engine_h20_hours_lower_bound": 0.2, "score_req_s_per_gpu": None},
{"trial_id": "t3", "engine_h20_hours_lower_bound": 0.3, "score_req_s_per_gpu": 9.6},
]
curve = analysis.sequential_curve(trials, 10.0, [0.05, 0.04])
assert curve["cost_to_threshold"]["regret_le_0.05"]["trial_id"] == "t3"
assert curve["cost_to_threshold"]["regret_le_0.04"]["trial_id"] == "t3"
assert math.isclose(curve["total_engine_h20_hours_lower_bound"], 0.6)
candidates = analysis.tie_expanded_candidates(
{"a": 3.0, "b": 2.0, "c": 2.0, "d": 1.0}, 2
)
assert candidates == ["a", "b", "c"]
policy = analysis.real_final_policy(
candidates,
{"a": 1.0, "b": 4.0, "c": 2.0, "d": 3.0},
{"a": 0.1, "b": 0.2, "c": 0.3, "d": 0.4},
)
assert policy["selected_cell"] == "b"
assert policy["real_regret"] == 0.0
assert math.isclose(policy["engine_h20_hours_lower_bound"], 0.6)
print("tuning cost analysis: PASS")
if __name__ == "__main__":
main()