#!/usr/bin/env python3 """Reconstruct tuning cost and cost-to-oracle curves from existing runs. The historical engine logs do not contain controller setup/cleanup timestamps. Consequently the reported GPU cost is an engine-lifetime lower bound: parallel_size * (last engine timestamp - first engine timestamp). It must not be presented as all-in method cost. New experiments should record allocation start/end directly. """ from __future__ import annotations import argparse import hashlib import json import math import re import subprocess from datetime import datetime from pathlib import Path from typing import Any from urllib.parse import urlsplit SCHEMA = "aituner-tuning-cost-analysis-v1" TIMESTAMP = re.compile(r"\b(\d{2}-\d{2} \d{2}:\d{2}:\d{2})(?:\.\d+)?\b") def numeric_summary(values: list[float]) -> dict[str, Any]: values = [float(value) for value in values] return { "n": len(values), "min": min(values) if values else None, "max": max(values) if values else None, "distinct_n": len(set(values)), } def sha256_text(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() class Reader: def __init__(self, repo_root: Path): self.repo_root = repo_root def _split(self, locator: str) -> tuple[str | None, str]: if locator.startswith("ssh://"): parsed = urlsplit(locator) if not parsed.hostname or not parsed.path.startswith("/"): raise ValueError(f"invalid SSH locator: {locator}") return parsed.hostname, parsed.path path = Path(locator) if not path.is_absolute(): path = self.repo_root / path return None, str(path) def read_text(self, locator: str) -> str: host, path = self._split(locator) if host is None: return Path(path).read_text(encoding="utf-8", errors="replace") completed = subprocess.run( ["ssh", host, "cat", "--", path], check=True, capture_output=True, text=True, ) return completed.stdout def join_locator(root: str, *parts: str) -> str: return "/".join([root.rstrip("/"), *(part.strip("/") for part in parts)]) def timestamp_span(log_text: str, year: int) -> tuple[float | None, bool, int]: parsed = [ datetime.strptime(f"{year}-{match}", "%Y-%m-%d %H:%M:%S") for match in TIMESTAMP.findall(log_text) ] if not parsed: return None, True, 0 monotonic = all(right >= left for left, right in zip(parsed, parsed[1:])) return (max(parsed) - min(parsed)).total_seconds(), monotonic, len(parsed) def load_campaign( reader: Reader, root: str, year: int, missing_duration_s: dict[str, float] | None = None, missing_reason: dict[str, str] | None = None, ) -> dict[str, Any]: missing_duration_s = missing_duration_s or {} missing_reason = missing_reason or {} state_text = reader.read_text(join_locator(root, "state.json")) state = json.loads(state_text) trials = [] for trial in state["trials"]: trial_id = str(trial["trial_id"]) log_text = reader.read_text(join_locator(root, "trials", trial_id, "engine.log")) duration_s, monotonic, timestamp_n = timestamp_span(log_text, year) duration_source = "engine_log_span" duration_note = "" if duration_s is None: duration_s = float(missing_duration_s.get(trial_id, 0.0)) duration_source = ( "conservative_fallback" if trial_id in missing_duration_s else "no_engine_timestamp" ) duration_note = missing_reason.get(trial_id, "") parallel_size = int(trial["parallel_size"]) score = trial.get("best_request_rate_per_gpu") trials.append( { "trial_id": trial_id, "status": trial["status"], "failure_stage": trial.get("failure_stage", ""), "parallel_size": parallel_size, "score_req_s_per_gpu": None if score is None else float(score), "config_patch": trial["config_patch"], "duration_s": duration_s, "duration_source": duration_source, "duration_note": duration_note, "engine_timestamp_n": timestamp_n, "engine_timestamps_monotonic": monotonic, "engine_h20_hours_lower_bound": duration_s * parallel_size / 3600.0, "engine_log_sha256": sha256_text(log_text), } ) return { "root": root, "state_sha256": sha256_text(state_text), "trials": trials, } def fixed_task_context(spec: dict[str, Any]) -> dict[str, Any]: flags = dict(spec["engine"]["base_flags"]) flags.pop("port", None) return { "model": spec["model"], "hardware": spec["hardware"], "engine_version": spec["engine"]["engine_version"], "launch_args": spec["engine"]["launch_args"], "base_flags_without_port": flags, "search": spec["search"], "slo": spec["slo"], "trace": spec["trace"], } def regret(score: float, reference: float) -> float: if reference <= 0: raise ValueError("reference score must be positive") return max(0.0, 1.0 - float(score) / float(reference)) def sequential_curve( trials: list[dict[str, Any]], reference: float, thresholds: list[float] ) -> dict[str, Any]: cumulative_cost = 0.0 best_score: float | None = None points = [] for trial in trials: cumulative_cost += float(trial["engine_h20_hours_lower_bound"]) score = trial["score_req_s_per_gpu"] if score is not None: best_score = score if best_score is None else max(best_score, score) points.append( { "trial_id": trial["trial_id"], "cumulative_engine_h20_hours_lower_bound": cumulative_cost, "best_score_req_s_per_gpu": best_score, "regret": None if best_score is None else regret(best_score, reference), } ) hits = {} for threshold in thresholds: hit = next( ( point for point in points if point["regret"] is not None and float(point["regret"]) <= float(threshold) + 1e-12 ), None, ) hits[f"regret_le_{threshold:g}"] = hit return { "reference_score_req_s_per_gpu": reference, "points": points, "cost_to_threshold": hits, "total_engine_h20_hours_lower_bound": cumulative_cost, } def surface_cell(trial: dict[str, Any]) -> str: flags = trial["config_patch"]["flag_patch"] return f"tp{int(flags['tensor-parallel-size'])}_mns{int(flags['max-num-seqs'])}" def tie_expanded_candidates(scores: dict[str, float], nominal_k: int) -> list[str]: ordered = sorted(scores, key=lambda cell: (-float(scores[cell]), cell)) if nominal_k <= 0 or nominal_k > len(ordered): raise ValueError("nominal k outside score surface") cutoff = float(scores[ordered[nominal_k - 1]]) tolerance = max(1e-12, abs(cutoff) * 1e-12) return [cell for cell in ordered if float(scores[cell]) >= cutoff - tolerance] def real_final_policy( candidates: list[str], real_scores: dict[str, float], cell_costs: dict[str, float] ) -> dict[str, Any]: oracle = max(real_scores.values()) selected = max(candidates, key=lambda cell: (real_scores[cell], cell)) return { "candidate_cells": candidates, "real_evaluations": len(candidates), "selected_cell": selected, "selected_real_score_req_s_per_gpu": real_scores[selected], "real_regret": regret(real_scores[selected], oracle), "engine_h20_hours_lower_bound": sum(cell_costs[cell] for cell in candidates), } def percentage_saving(new: float, old: float) -> float: if old <= 0: raise ValueError("baseline cost must be positive") return 1.0 - new / old def build_analysis(manifest_path: Path) -> dict[str, Any]: repo_root = manifest_path.resolve().parents[2] manifest = json.loads(manifest_path.read_text(encoding="utf-8")) reader = Reader(repo_root) year = int(manifest["year"]) sequential = {} task_contexts = {} for name, run in manifest["sequential_runs"].items(): sequential[name] = load_campaign( reader, run["root"], year, run.get("missing_log_duration_s"), run.get("missing_log_reason"), ) spec_text = reader.read_text(join_locator(run["root"], "study_spec.snapshot.json")) sequential[name]["study_spec_sha256"] = sha256_text(spec_text) task_contexts[name] = fixed_task_context(json.loads(spec_text)) all_sequential_scores = [ trial["score_req_s_per_gpu"] for campaign in sequential.values() for trial in campaign["trials"] if trial["score_req_s_per_gpu"] is not None ] empirical_reference = max(all_sequential_scores) thresholds = [float(value) for value in manifest["threshold_regrets"]] for campaign in sequential.values(): campaign["curve"] = sequential_curve( campaign["trials"], empirical_reference, thresholds ) surface_manifest = manifest["real_surface"] primary = load_campaign(reader, surface_manifest["primary_root"], year) companion = load_campaign(reader, surface_manifest["tp4_companion_root"], year) completed_surface_trials = [ trial for trial in primary["trials"] + companion["trials"] if trial["status"] == "completed" and trial["score_req_s_per_gpu"] is not None ] cells: dict[str, dict[str, Any]] = {} for trial in completed_surface_trials: flags = trial["config_patch"]["flag_patch"] if int(flags["max-num-batched-tokens"]) != int( surface_manifest["fixed_max_num_batched_tokens"] ): raise ValueError("surface MBT invariant failed") cell = surface_cell(trial) if cell in cells: raise ValueError(f"duplicate completed surface cell: {cell}") cells[cell] = trial real_scores = {cell: float(trial["score_req_s_per_gpu"]) for cell, trial in cells.items()} cell_costs = { cell: float(trial["engine_h20_hours_lower_bound"]) for cell, trial in cells.items() } surface_oracle_score = max(real_scores.values()) surface_oracle_cells = [ cell for cell, score in real_scores.items() if math.isclose(score, surface_oracle_score) ] simulator_text = reader.read_text(manifest["simulator_metrics"]) simulator = json.loads(simulator_text) simulator_real_scores = { cell: float(score) for cell, score in simulator["real_scores"].items() } surface_matches_simulator = set(real_scores) == set(simulator_real_scores) and all( math.isclose(real_scores[cell], simulator_real_scores[cell], abs_tol=1e-12) for cell in real_scores ) throughput = simulator["analyses"]["frozen-calibrated/throughput-proxy"] throughput_scores = { cell: float(score) for cell, score in throughput["simulated_scores"].items() } throughput_real_final = { f"nominal_k_{k}": real_final_policy( tie_expanded_candidates(throughput_scores, k), real_scores, cell_costs ) for k in (1, 2, 3) } throughput_top1 = tie_expanded_candidates(throughput_scores, 1) simulator_only_cell = throughput_top1[0] slo = simulator["analyses"]["frozen-calibrated/SLO-gated"] slo_top_bucket = list(slo["metrics"]["top1"]["candidate_cells"]) slo_diagnostic = real_final_policy(slo_top_bucket, real_scores, cell_costs) pure_hits = sequential["pure_llm"]["curve"]["cost_to_threshold"] guided_hits = sequential["guided_harness"]["curve"]["cost_to_threshold"] direct_comparison = {} for threshold in (0.05, 0.02): key = f"regret_le_{threshold:g}" pure_hit = pure_hits[key] guided_hit = guided_hits[key] if pure_hit is None or guided_hit is None: continue pure_cost = float(pure_hit["cumulative_engine_h20_hours_lower_bound"]) guided_cost = float(guided_hit["cumulative_engine_h20_hours_lower_bound"]) direct_comparison[key] = { "pure_llm_h20_hours_lower_bound": pure_cost, "guided_harness_h20_hours_lower_bound": guided_cost, "guided_saving_vs_pure_llm": percentage_saving(guided_cost, pure_cost), } five_cost = direct_comparison["regret_le_0.05"][ "guided_harness_h20_hours_lower_bound" ] two_cost = direct_comparison["regret_le_0.02"][ "guided_harness_h20_hours_lower_bound" ] sim_real_cost = float(slo_diagnostic["engine_h20_hours_lower_bound"]) target_bars = { "five_percent_regret": { "twenty_percent_below_current_guided": 0.8 * five_cost, "thirty_percent_below_posthoc_sim_slo_real_final": 0.7 * sim_real_cost, "development_target_h20_hours_lower_bound": min( 0.8 * five_cost, 0.7 * sim_real_cost ), }, "two_percent_regret": { "twenty_percent_below_current_guided": 0.8 * two_cost, "thirty_percent_below_posthoc_sim_slo_real_final": 0.7 * sim_real_cost, "development_target_h20_hours_lower_bound": min( 0.8 * two_cost, 0.7 * sim_real_cost ), }, } all_trial_costs = [ float(trial["engine_h20_hours_lower_bound"]) for campaign in sequential.values() for trial in campaign["trials"] ] + [ float(trial["engine_h20_hours_lower_bound"]) for campaign in (primary, companion) for trial in campaign["trials"] ] all_regrets = [ float(point["regret"]) for campaign in sequential.values() for point in campaign["curve"]["points"] if point["regret"] is not None ] monotonic_logs = all( trial["engine_timestamps_monotonic"] for campaign in [*sequential.values(), primary, companion] for trial in campaign["trials"] ) invariants = { "dash0_task_contexts_equal_except_method_and_port": len( {json.dumps(value, sort_keys=True) for value in task_contexts.values()} ) == 1, "surface_has_expected_cell_count": len(cells) == int(surface_manifest["expected_cells"]), "surface_matches_simulator_real_scores": surface_matches_simulator, "all_costs_non_negative": all(value >= 0 for value in all_trial_costs), "all_regrets_in_0_1": all(0 <= value <= 1 for value in all_regrets), "surface_scores_not_all_identical": len(set(real_scores.values())) > 1, "sequential_scores_not_all_identical": len(set(all_sequential_scores)) > 1, "engine_log_timestamps_monotonic": monotonic_logs, "sequential_trial_counts_match_manifest": all( len(sequential[name]["trials"]) == int(run["expected_trials"]) for name, run in manifest["sequential_runs"].items() ), "simulator_suite_has_no_failed_runs": int(simulator["execution"]["failed_runs"]) == 0, "simulator_scores_not_all_identical": len(set(throughput_scores.values())) > 1, } failed_invariants = [name for name, passed in invariants.items() if not passed] if failed_invariants: raise RuntimeError(f"data sanity invariant failed: {failed_invariants}") failed_primary_attempts = [ trial for trial in primary["trials"] if trial["status"] != "completed" ] return { "schema": SCHEMA, "cost_definition": { "reported_metric": "engine H20-hours lower bound", "formula": "parallel_size * (last_engine_log_timestamp - first_engine_log_timestamp) / 3600", "included": ["engine startup after first timestamp", "warm-up/probes until last timestamp"], "not_reconstructable": [ "GPU allocation before first engine timestamp", "controller/LLM latency", "cleanup after last engine timestamp", "one-time simulator operator profiling GPU-hours", ], "future_all_in_metric": "allocation_start_to_GPU_idle * allocated_GPU_count, including failures", }, "comparison_scope": { "pure_llm_vs_guided_harness": "direct: same dash0 fixed task context", "simulator_vs_surface": "direct: simulator predictions and exact dash1 12-cell real surface", "dash0_methods_vs_dash1_simulator": "indicative only: matched model/engine/workload/GPU type, different host and campaign", }, "empirical_reference": { "score_req_s_per_gpu": empirical_reference, "meaning": "best observed across the two dash0 sequential runs; not a global oracle", }, "sequential_runs": sequential, "direct_dash0_comparison": direct_comparison, "real_surface": { "cells": { cell: { "score_req_s_per_gpu": real_scores[cell], "engine_h20_hours_lower_bound": cell_costs[cell], } for cell in sorted(cells) }, "oracle_score_req_s_per_gpu": surface_oracle_score, "oracle_cells": surface_oracle_cells, "completed_annotation_engine_h20_hours_lower_bound": sum(cell_costs.values()), "failed_primary_attempt_n": len(failed_primary_attempts), "failed_primary_attempt_engine_h20_hours_lower_bound": sum( float(trial["engine_h20_hours_lower_bound"]) for trial in failed_primary_attempts ), "primary_state_sha256": primary["state_sha256"], "tp4_companion_state_sha256": companion["state_sha256"], }, "simulator": { "marginal_gpu_hours_without_real_verification": 0.0, "observed_fidelity_suite_cpu_hours": float( simulator["execution"]["suite_elapsed_seconds"] ) / 3600.0, "observed_fidelity_suite_runs": int(simulator["execution"]["attempted_runs"]), "one_time_profile_gpu_hours": None, "one_time_profile_cost_status": "not recorded; total cold-start cost is unknown", "decision_bearing_throughput_proxy_sim_only_top1": { "selected_cell": simulator_only_cell, "selected_real_score_req_s_per_gpu": real_scores[simulator_only_cell], "real_regret": regret(real_scores[simulator_only_cell], surface_oracle_score), "gpu_hours": 0.0, }, "decision_bearing_throughput_proxy_plus_real_final": throughput_real_final, "posthoc_slo_gated_plus_real_final": { **slo_diagnostic, "status": "diagnostic/post-hoc, not a preregistered prospective policy", "false_feasible": int(slo["false_feasibility"]["overall"]["false_feasible"]), "false_infeasible": int(slo["false_feasibility"]["overall"]["false_infeasible"]), }, "metrics_sha256": sha256_text(simulator_text), }, "provisional_development_targets": { "status": "lower-bound, single-task targets; require same-host prospective validation", **target_bars, }, "data_sanity": { "invariants": invariants, "sequential_score_summary": numeric_summary(all_sequential_scores), "surface_score_summary": numeric_summary(list(real_scores.values())), "simulator_throughput_score_summary": numeric_summary( list(throughput_scores.values()) ), "trial_cost_summary": numeric_summary(all_trial_costs), "regret_summary": numeric_summary(all_regrets), }, } def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--manifest", type=Path, default=Path(__file__).with_name("manifest.json")) parser.add_argument("--output", type=Path, default=Path(__file__).with_name("metrics.json")) args = parser.parse_args() analysis = build_analysis(args.manifest) args.output.write_text(json.dumps(analysis, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(json.dumps({ "status": "ok", "output": str(args.output), "empirical_reference": analysis["empirical_reference"], "surface_oracle": analysis["real_surface"]["oracle_score_req_s_per_gpu"], "sanity": analysis["data_sanity"], }, indent=2, sort_keys=True)) if __name__ == "__main__": main()