Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)
Workload-conditioned operator profiling on patched vLLM 0.24.0 + Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64 raggedness, 8-45% token-efficiency loss vs rectangular controls); mechanism decomposition kills the padding narrative and finds the arrival-uniformization artifact (-12.9%); cross-version churn surface shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held. Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer sidecars and metrics are tracked. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
466
runs/opprof-phase6/analyze_phase6.py
Normal file
466
runs/opprof-phase6/analyze_phase6.py
Normal file
@@ -0,0 +1,466 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frozen Phase-6 paired-anchor, frontier, rank, and Layer-1 analysis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
OLD_BEST = "tp2_mns32"
|
||||
TOP20 = {"tp2_mns32", "tp2_mns64", "tp4_mns16", "tp4_mns32", "tp4_mns64"}
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def numeric(values: list[float | int | None]) -> dict[str, Any]:
|
||||
finite = [float(x) for x in values if x is not None and math.isfinite(float(x))]
|
||||
return {
|
||||
"n": len(values), "finite_n": len(finite), "missing_n": len(values) - len(finite),
|
||||
"min": min(finite) if finite else None, "max": max(finite) if finite else None,
|
||||
"distinct_n": len(set(finite)),
|
||||
}
|
||||
|
||||
|
||||
def cv(values: list[float]) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
array = np.asarray(values, dtype=np.float64)
|
||||
mean = float(array.mean())
|
||||
return float(array.std(ddof=0) / mean) if mean else 0.0
|
||||
|
||||
|
||||
def floor_buckets(scores: dict[str, float]) -> tuple[float, dict[str, int]]:
|
||||
tol = max(1e-9, 1e-6 * max(abs(x) for x in scores.values()))
|
||||
return tol, {key: math.floor(value / tol) for key, value in scores.items()}
|
||||
|
||||
|
||||
def kendall_tau_b(x: dict[str, int], y: dict[str, int]) -> dict[str, Any]:
|
||||
keys = sorted(x)
|
||||
c = d = tx = ty = joint = 0
|
||||
for i, left in enumerate(keys):
|
||||
for right in keys[i + 1:]:
|
||||
dx = (x[left] > x[right]) - (x[left] < x[right])
|
||||
dy = (y[left] > y[right]) - (y[left] < y[right])
|
||||
if dx == 0 and dy == 0:
|
||||
joint += 1
|
||||
elif dx == 0:
|
||||
tx += 1
|
||||
elif dy == 0:
|
||||
ty += 1
|
||||
elif dx == dy:
|
||||
c += 1
|
||||
else:
|
||||
d += 1
|
||||
denom = math.sqrt((c + d + tx) * (c + d + ty))
|
||||
return {
|
||||
"tau_b": (c - d) / denom if denom else None, "concordant": c,
|
||||
"discordant": d, "v20_only_ties": tx, "v24_only_ties": ty,
|
||||
"joint_ties": joint, "pairs": len(keys) * (len(keys) - 1) // 2,
|
||||
}
|
||||
|
||||
|
||||
def layer_metrics(records: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
model = [x for x in records if x.get("model_executed")]
|
||||
mode = Counter(str(x["cudagraph"]["runtime_mode"]) for x in model)
|
||||
total_bucket = sum(int(x["cudagraph"]["bucket_tokens"]) for x in model)
|
||||
total_padding = sum(int(x["cudagraph"]["padding_tokens"]) for x in model)
|
||||
kv = [float(x["kv"]["usage"]) for x in model]
|
||||
waiting = [float(x["queues"]["waiting"]) for x in model]
|
||||
decode_b = [float(x["decode_batch_size"]) for x in model]
|
||||
return {
|
||||
"steps": len(records), "model_steps": len(model),
|
||||
"prefill_only_steps": sum(int(x["prefill_tokens"]) > 0 and int(x["decode_tokens"]) == 0 for x in model),
|
||||
"decode_only_steps": sum(int(x["prefill_tokens"]) == 0 and int(x["decode_tokens"]) > 0 for x in model),
|
||||
"mixed_steps": sum(int(x["prefill_tokens"]) > 0 and int(x["decode_tokens"]) > 0 for x in model),
|
||||
"prefill_tokens": sum(int(x["prefill_tokens"]) for x in model),
|
||||
"decode_tokens": sum(int(x["decode_tokens"]) for x in model),
|
||||
"preemptions": sum(int(x["preemptions"]) for x in model),
|
||||
"kv_usage_mean": float(np.mean(kv)) if kv else None,
|
||||
"kv_usage_max": max(kv) if kv else None,
|
||||
"waiting_mean": float(np.mean(waiting)) if waiting else None,
|
||||
"waiting_max": max(waiting) if waiting else None,
|
||||
"waiting_cv": cv(waiting), "decode_B_mean": float(np.mean(decode_b)) if decode_b else None,
|
||||
"decode_B_cv": cv(decode_b), "graph_mode_counts": dict(sorted(mode.items())),
|
||||
"graph_mode_shares": {key: value / len(model) for key, value in sorted(mode.items())} if model else {},
|
||||
"padding_fraction": total_padding / total_bucket if total_bucket else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def load_stream(path: Path) -> list[dict[str, Any]]:
|
||||
return [json.loads(line) for line in path.read_text().splitlines() if "step_index" in json.loads(line)]
|
||||
|
||||
|
||||
def p3_reference(p3_root: Path) -> dict[str, Any]:
|
||||
run = p3_root / "primary/P10-C00/moderate"
|
||||
result = json.loads((run / "client/result.json").read_text())
|
||||
t0 = int(result["t0_mono_ns"])
|
||||
lo = t0 + int(float(result["clean"]["start_s"]) * 1e9)
|
||||
hi = t0 + int(float(result["clean"]["end_s"]) * 1e9)
|
||||
stream = next((run / "opprof").glob("*.jsonl"))
|
||||
records = [x for x in load_stream(stream) if lo <= int(x["submit_mono_ns"]) < hi]
|
||||
return {
|
||||
"limitation": "P3 P10 reference is TP1/MNS-default, steady arrival, sampling_u<=0.125, output<=256; it is contextual rather than a matched v0.20 composition baseline.",
|
||||
"stream_sha256": sha256_file(stream), "metrics": layer_metrics(records),
|
||||
}
|
||||
|
||||
|
||||
def accepted_anchor(anchor_dir: Path) -> dict[str, Any]:
|
||||
primary = json.loads((anchor_dir / "result.json").read_text())
|
||||
cell_dir = anchor_dir.parent
|
||||
anchor = float(primary["anchor"])
|
||||
confirms = []
|
||||
for path in sorted(cell_dir.glob("confirm-*-anchor-*/result.json")):
|
||||
item = json.loads(path.read_text())
|
||||
if math.isclose(float(item["anchor"]), anchor, rel_tol=0, abs_tol=1e-15):
|
||||
confirms.append(item)
|
||||
trials = [primary, *confirms]
|
||||
votes = [bool(x["feasible"]) for x in trials]
|
||||
if len(votes) == 1 or len(set(votes)) == 1:
|
||||
verdict = votes[0]
|
||||
resolved = True
|
||||
elif len(votes) >= 3:
|
||||
verdict = sum(votes) >= 2
|
||||
resolved = True
|
||||
else:
|
||||
verdict = None
|
||||
resolved = False
|
||||
return {
|
||||
"anchor": anchor, "primary": primary, "confirmations": confirms,
|
||||
"trial_count": len(trials), "pass_rates": [x["pass_rate"] for x in trials],
|
||||
"feasible_votes": votes, "accepted_feasible": verdict, "resolved": resolved,
|
||||
"accepted_pass_rate": float(np.median([x["pass_rate"] for x in trials])),
|
||||
}
|
||||
|
||||
|
||||
def matching_anchor_dir(cell_dir: Path, anchor: float) -> Path | None:
|
||||
for path in cell_dir.glob("anchor-*/result.json"):
|
||||
item = json.loads(path.read_text())
|
||||
if math.isclose(float(item["anchor"]), anchor, rel_tol=0, abs_tol=1e-15):
|
||||
return path.parent
|
||||
return None
|
||||
|
||||
|
||||
def analyze(root: Path, ground_path: Path, p3_root: Path) -> dict[str, Any]:
|
||||
ground = json.loads(ground_path.read_text())
|
||||
old_cells = {x["cell_id"]: x for x in ground["cells"]}
|
||||
old_probe = {
|
||||
(x["cell_id"], float(p["sampling_u"])): p
|
||||
for x in ground["cells"] for p in x["probe_history"]
|
||||
}
|
||||
cells: dict[str, Any] = {}
|
||||
all_primary = []
|
||||
all_client_invariants = []
|
||||
selection_matches = []
|
||||
for cell, old in old_cells.items():
|
||||
cell_dir = root / "solo-authoritative/cells" / cell
|
||||
colocated_dir = root / "cells" / cell
|
||||
f20 = float(old["measured_objective"]["slo_feasible_req_s_per_gpu"])
|
||||
if not (cell_dir / "cell-valid.json").exists():
|
||||
cells[cell] = {
|
||||
"tp": old["tensor_parallel_size"], "mns": old["max_num_seqs"],
|
||||
"measurement_status": "UNMEASURED_SOLO",
|
||||
"f20": f20, "f24": None, "drift": None,
|
||||
"observed_max_feasible_rate": None,
|
||||
"material_frontier_moved": None, "boundary": "UNMEASURED",
|
||||
"bounded": False, "censor": "UNMEASURED_SOLO",
|
||||
"anchors": [], "cell_valid": None,
|
||||
}
|
||||
continue
|
||||
valid = json.loads((cell_dir / "cell-valid.json").read_text())
|
||||
stream = next((cell_dir / "opprof").glob("*.jsonl"))
|
||||
stream_records = load_stream(stream)
|
||||
anchors = []
|
||||
for path in sorted(cell_dir.glob("anchor-*/result.json")):
|
||||
accepted = accepted_anchor(path.parent)
|
||||
primary = accepted["primary"]
|
||||
key = (cell, float(primary["anchor"]))
|
||||
historical = old_probe[key]
|
||||
lo = int(primary["interval"]["start_mono_ns"])
|
||||
hi = int(primary["interval"]["end_mono_ns"])
|
||||
records = [x for x in stream_records if lo <= int(x["submit_mono_ns"]) <= hi]
|
||||
accepted["layer1"] = layer_metrics(records)
|
||||
for confirmation in accepted["confirmations"]:
|
||||
confirm_lo = int(confirmation["interval"]["start_mono_ns"])
|
||||
confirm_hi = int(confirmation["interval"]["end_mono_ns"])
|
||||
confirmation["layer1"] = layer_metrics([
|
||||
x for x in stream_records
|
||||
if confirm_lo <= int(x["submit_mono_ns"]) <= confirm_hi
|
||||
])
|
||||
accepted["v20"] = {
|
||||
"pass_rate": historical["pass_rate"], "feasible": historical["feasible"],
|
||||
"request_count": historical["request_count"],
|
||||
"rate_per_gpu": historical["request_rate_per_gpu_req_s_gpu"],
|
||||
}
|
||||
accepted["v24"] = {
|
||||
"pass_rate": accepted["accepted_pass_rate"],
|
||||
"feasible": accepted["accepted_feasible"],
|
||||
"request_count": primary["selection"]["count"],
|
||||
"rate_per_gpu": primary["selection"]["offered_req_s_per_gpu"],
|
||||
}
|
||||
colocated_anchor_dir = matching_anchor_dir(colocated_dir, float(primary["anchor"]))
|
||||
if colocated_anchor_dir is not None:
|
||||
colocated = accepted_anchor(colocated_anchor_dir)
|
||||
accepted["colocated"] = {
|
||||
"primary_pass_rate": colocated["primary"]["pass_rate"],
|
||||
"primary_feasible": colocated["primary"]["feasible"],
|
||||
"trial_pass_rates": colocated["pass_rates"],
|
||||
"accepted_feasible": colocated["accepted_feasible"],
|
||||
"solo_minus_colocated_primary_pass_rate": (
|
||||
accepted["accepted_pass_rate"] - colocated["primary"]["pass_rate"]
|
||||
),
|
||||
}
|
||||
else:
|
||||
accepted["colocated"] = None
|
||||
accepted["feasibility_flip"] = (
|
||||
accepted["accepted_feasible"] is not None
|
||||
and accepted["accepted_feasible"] != historical["feasible"]
|
||||
)
|
||||
anchors.append(accepted)
|
||||
all_primary.append(primary)
|
||||
all_client_invariants.extend(primary["invariants"].values())
|
||||
selection_matches.append(primary["selection"]["count"] == historical["request_count"])
|
||||
peak_u = float(old["measured_objective"]["best_sampling_u"])
|
||||
peak = next(x for x in anchors if math.isclose(x["anchor"], peak_u, abs_tol=1e-15))
|
||||
feasible = [x for x in anchors if x["accepted_feasible"] is True]
|
||||
infeasible = [x for x in anchors if x["accepted_feasible"] is False]
|
||||
unresolved = [x for x in anchors if x["accepted_feasible"] is None]
|
||||
observed_frontier = max((x["v24"]["rate_per_gpu"] for x in feasible), default=None)
|
||||
lower = [x for x in anchors if x["anchor"] < peak_u]
|
||||
upper = [x for x in anchors if x["anchor"] > peak_u]
|
||||
if unresolved:
|
||||
bounded, censor = False, "UNRESOLVED_SOLO_ANCHOR"
|
||||
elif feasible and infeasible:
|
||||
monotonic = max(x["anchor"] for x in feasible) < min(x["anchor"] for x in infeasible)
|
||||
bounded = monotonic
|
||||
censor = None if bounded else "NONMONOTONIC_SOLO_ANCHORS"
|
||||
elif feasible:
|
||||
bounded, censor = False, "RIGHT_CENSORED_HISTORY_EDGE"
|
||||
elif infeasible:
|
||||
bounded, censor = False, "LEFT_CENSORED_HISTORY_EDGE"
|
||||
else:
|
||||
bounded, censor = False, "NO_RESOLVED_SOLO_ANCHOR"
|
||||
frontier = (
|
||||
None if censor in {
|
||||
"UNRESOLVED_SOLO_ANCHOR", "NONMONOTONIC_SOLO_ANCHORS",
|
||||
"NO_RESOLVED_SOLO_ANCHOR",
|
||||
} else observed_frontier
|
||||
)
|
||||
drift = (frontier / f20 - 1) if frontier is not None else None
|
||||
if censor == "NONMONOTONIC_SOLO_ANCHORS":
|
||||
boundary = "NONMONOTONIC"
|
||||
elif peak["accepted_feasible"] is False:
|
||||
boundary = "DOWN"
|
||||
elif peak["accepted_feasible"] is None:
|
||||
boundary = "UNRESOLVED"
|
||||
elif any(x["accepted_feasible"] is True for x in upper):
|
||||
boundary = "UP"
|
||||
else:
|
||||
boundary = "STABLE"
|
||||
cells[cell] = {
|
||||
"tp": old["tensor_parallel_size"], "mns": old["max_num_seqs"],
|
||||
"measurement_status": "SOLO_AUTHORITATIVE",
|
||||
"f20": f20, "f24": frontier, "drift": drift,
|
||||
"observed_max_feasible_rate": observed_frontier,
|
||||
"material_frontier_moved": bounded and drift is not None and abs(drift) > .05,
|
||||
"boundary": boundary, "bounded": bounded, "censor": censor,
|
||||
"anchors": anchors, "cell_valid": valid,
|
||||
}
|
||||
|
||||
scores20 = {key: value["f20"] for key, value in cells.items()}
|
||||
scores24 = {key: value["f24"] for key, value in cells.items() if value["f24"] is not None}
|
||||
tol20, buckets20 = floor_buckets(scores20)
|
||||
tol24, buckets24 = floor_buckets(scores24)
|
||||
full_bounded = len(scores24) == 12 and all(x["bounded"] for x in cells.values())
|
||||
max24 = max(buckets24.values())
|
||||
if not full_bounded:
|
||||
argmax = "INCONCLUSIVE"
|
||||
else:
|
||||
argmax = "SURVIVED" if buckets24[OLD_BEST] == max24 else "MOVED"
|
||||
tau = kendall_tau_b(buckets20, buckets24) if full_bounded else None
|
||||
reversals = []
|
||||
for left in sorted(TOP20):
|
||||
for right in sorted(TOP20):
|
||||
if left >= right:
|
||||
continue
|
||||
if not cells[left]["bounded"] or not cells[right]["bounded"]:
|
||||
continue
|
||||
if left not in scores24 or right not in scores24:
|
||||
continue
|
||||
old_delta = scores20[left] - scores20[right]
|
||||
if abs(old_delta) / max(scores20[left], scores20[right]) <= .05:
|
||||
continue
|
||||
new_delta = scores24[left] - scores24[right]
|
||||
if old_delta * new_delta < 0:
|
||||
reversals.append([left, right])
|
||||
if not full_bounded or argmax == "INCONCLUSIVE" or tau is None:
|
||||
ranking = "INCONCLUSIVE"
|
||||
elif argmax == "MOVED" or tau["tau_b"] < .8 or reversals:
|
||||
ranking = "MOVED"
|
||||
else:
|
||||
ranking = "SURVIVED"
|
||||
trap_inputs = ["tp4_mns8", "tp4_mns16", "tp4_mns32"]
|
||||
if not all(cells[x]["bounded"] for x in trap_inputs):
|
||||
trap = "INCONCLUSIVE"
|
||||
elif buckets24["tp4_mns16"] == max24:
|
||||
trap = "CEASES_TO_BE_A_TRAP"
|
||||
elif buckets24["tp4_mns16"] >= max(buckets24["tp4_mns8"], buckets24["tp4_mns32"]):
|
||||
trap = "PERSISTS"
|
||||
else:
|
||||
trap = "ESCAPES"
|
||||
|
||||
colocated_state = json.loads((root / "controller-state.json").read_text())
|
||||
solo_state_path = root / "solo-authoritative/controller-state.json"
|
||||
solo_state = json.loads(solo_state_path.read_text()) if solo_state_path.exists() else None
|
||||
state = solo_state or colocated_state
|
||||
measured_cells = [
|
||||
x for x in cells.values() if x["measurement_status"] == "SOLO_AUTHORITATIVE"
|
||||
]
|
||||
coverage = {
|
||||
"solo_primary_anchors_at_least_25": len(all_primary) >= 25,
|
||||
"solo_measured_cells_12": len(measured_cells) == 12,
|
||||
}
|
||||
invariants = {
|
||||
"surface_rows_12": len(cells) == 12,
|
||||
"selection_counts_match": all(selection_matches),
|
||||
"client_invariants": all(all_client_invariants),
|
||||
"cell_validity": all(
|
||||
all(x["cell_valid"]["invariants"].values()) for x in measured_cells
|
||||
),
|
||||
"gpu_below_6": float(state["gpu_hours_total"]) < 6.0,
|
||||
"rates_nonnegative": all(x["f24"] is None or x["f24"] >= 0 for x in cells.values()),
|
||||
"surface_not_identical": len(set(scores24.values())) > 1,
|
||||
}
|
||||
red_flags = [key for key, value in {**coverage, **invariants}.items() if not value]
|
||||
drifted = [key for key, value in cells.items() if value["material_frontier_moved"] is True]
|
||||
flip_cells = [
|
||||
key for key, value in cells.items()
|
||||
if any(anchor["feasibility_flip"] for anchor in value["anchors"])
|
||||
]
|
||||
partial = bool([key for key, value in coverage.items() if not value])
|
||||
w1_audit_path = root / "w1-readjudication-A-P6-1.json"
|
||||
w1_audit = json.loads(w1_audit_path.read_text()) if w1_audit_path.exists() else None
|
||||
censored = {key: value["censor"] for key, value in cells.items() if not value["bounded"]}
|
||||
colocated_deltas = [
|
||||
{
|
||||
"cell": cell, "anchor": anchor["anchor"],
|
||||
"solo_pass_rate": anchor["accepted_pass_rate"],
|
||||
"solo_feasible": anchor["accepted_feasible"],
|
||||
**anchor["colocated"],
|
||||
}
|
||||
for cell, value in cells.items() for anchor in value["anchors"]
|
||||
if anchor.get("colocated") is not None
|
||||
]
|
||||
return {
|
||||
"schema": 1,
|
||||
"status": "BUDGET_STOP_PARTIAL" if partial else ("VALID" if not red_flags else "INVALID"),
|
||||
"authoritative_tier": "A-P6-2 solo host placement",
|
||||
"limitation": "Upgrade-path churn includes dash1->dash0 and resolved-default changes. Co-located W1-W3 values are indicative only; P3 composition is contextual, not a matched v0.20 Layer-1 baseline.",
|
||||
"ground_truth_sha256": sha256_file(ground_path), "cells": cells,
|
||||
"floor_buckets": {"v20_tol": tol20, "v20": buckets20, "v24_tol": tol24, "v24": buckets24},
|
||||
"verdicts": {
|
||||
"argmax": argmax, "ranking": ranking, "trap": trap,
|
||||
"full_surface_bounded": full_bounded, "tau_b": tau,
|
||||
"top_pair_reversals_gt5pct": reversals,
|
||||
},
|
||||
"materially_drifted_cells": drifted,
|
||||
"feasibility_flip_cells": flip_cells,
|
||||
"decision_blockers": {
|
||||
"coverage": [key for key, value in coverage.items() if not value],
|
||||
"unbounded_or_unresolved_cells": censored,
|
||||
},
|
||||
"solo_vs_colocated": colocated_deltas,
|
||||
"w1_readjudication": w1_audit,
|
||||
"run_stats": {
|
||||
"measured_cells": len(measured_cells),
|
||||
"surface_cells": len(cells),
|
||||
"primary_anchor_runs": len(all_primary),
|
||||
"confirmation_runs": sum(
|
||||
len(anchor["confirmations"])
|
||||
for cell in cells.values() for anchor in cell["anchors"]
|
||||
),
|
||||
"accepted_anchor_trials": sum(
|
||||
anchor["trial_count"]
|
||||
for cell in cells.values() for anchor in cell["anchors"]
|
||||
),
|
||||
"warmup_runs": len(measured_cells),
|
||||
"solo_cell_gpu_hours": {
|
||||
key: value.get("gpu_hours") for key, value in (solo_state or {}).get("cells", {}).items()
|
||||
},
|
||||
"colocated_wave_gpu_hours": {
|
||||
key: value.get("gpu_hours") for key, value in colocated_state["waves"].items()
|
||||
},
|
||||
"launch_echo": (
|
||||
(root / "launch-echo.log").read_text().splitlines()
|
||||
+ ((root / "solo-authoritative/launch-echo.log").read_text().splitlines()
|
||||
if (root / "solo-authoritative/launch-echo.log").exists() else [])
|
||||
),
|
||||
},
|
||||
"attempt_history": {
|
||||
"colocated_status": colocated_state["status"],
|
||||
"colocated_h20_hours": colocated_state["gpu_hours_total"],
|
||||
"colocated_primary_anchors": colocated_state["completed_primary_anchors"],
|
||||
"colocated_confirmations": colocated_state["completed_confirmations"],
|
||||
"colocated_budget_stop": colocated_state.get("budget_stop"),
|
||||
"solo_status": (solo_state or {}).get("status"),
|
||||
"solo_repairs": (solo_state or {}).get("repairs", []),
|
||||
"solo_failures": (solo_state or {}).get("failures", []),
|
||||
"raw_roots": {
|
||||
"colocated": str(root / "cells"),
|
||||
"solo_authoritative": str(root / "solo-authoritative/cells"),
|
||||
},
|
||||
},
|
||||
"p3_composition_reference": p3_reference(p3_root),
|
||||
"gpu": {
|
||||
"new_h20_hours": state["gpu_hours_total"], "hard_cap": 6.0,
|
||||
"prior_colocated_h20_hours": colocated_state["gpu_hours_total"],
|
||||
"solo_h20_hours": (solo_state or {}).get("solo_gpu_hours", 0.0),
|
||||
"completed_primary_anchors": (solo_state or {}).get("primary_anchors", 0),
|
||||
"confirmations": (solo_state or {}).get("confirmations", 0),
|
||||
"controller_status": state["status"],
|
||||
"budget_stop": state.get("budget_stop"),
|
||||
},
|
||||
"sanity": {
|
||||
"red_flags": red_flags, "coverage": coverage, "invariants": invariants,
|
||||
"numeric": {
|
||||
"v20_score": numeric(list(scores20.values())),
|
||||
"v24_score": numeric(list(scores24.values())),
|
||||
"drift": numeric([x["drift"] for x in cells.values()]),
|
||||
"primary_pass_rate": numeric([x["pass_rate"] for x in all_primary]),
|
||||
"selected_count": numeric([x["selection"]["count"] for x in all_primary]),
|
||||
"layer1_steps": numeric([
|
||||
anchor["layer1"]["steps"] for cell in cells.values() for anchor in cell["anchors"]
|
||||
]),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--root", type=Path, required=True)
|
||||
p.add_argument("--ground-truth", type=Path, required=True)
|
||||
p.add_argument("--p3-root", type=Path, required=True)
|
||||
p.add_argument("--out", type=Path, required=True)
|
||||
args = p.parse_args()
|
||||
result = analyze(args.root, args.ground_truth, args.p3_root)
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n")
|
||||
print(json.dumps({"status": result["status"], "verdicts": result["verdicts"], "red_flags": result["sanity"]["red_flags"], "gpu": result["gpu"]}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user