gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden trace replay measurement integrity

Browse Source
This commit is contained in:
Gahow Wang
2026-07-13 17:17:30 +08:00
parent 5ae0525611
commit f56ecad64d
4 changed files with 138 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
scripts/collectivespec/summarize_fixed_grid.py
View File

@@ -31,7 +31,18 @@ def num(value: Any) -> float | None:
return value if math.isfinite(value) else None
def read_one(root: Path, k: int, expected_tokens: int | None) -> dict[str, Any]:
def nonnegative_int(value: Any) -> int | None:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
return None
return value
def read_one(
root: Path,
k: int,
expected_tokens: int | None,
expected_request_count: int | None,
) -> dict[str, Any]:
record: dict[str, Any] = {"k": k, "label": "NoSpec" if k == 0 else f"K={k}"}
store = root / "stores" / f"k{k}"
result_path = first(store, "*/trials/trial-*/result.json")
@@ -70,14 +81,36 @@ def read_one(root: Path, k: int, expected_tokens: int | None) -> dict[str, Any]:
probe = history[0]
details = detail_rows[0]
outcomes = details.get("outcomes") if isinstance(details.get("outcomes"), list) else []
request_count = int(probe.get("request_count", -1))
request_count = nonnegative_int(probe.get("request_count"))
replayed_request_count = nonnegative_int(probe.get("replayed_request_count"))
if request_count is None:
failures.append("invalid_request_count")
request_count = -1
if replayed_request_count is None:
failures.append("invalid_replayed_request_count")
replayed_request_count = -1
if details.get("early_stopped") or probe.get("early_stopped"):
failures.append("early_stopped")
if len(outcomes) != request_count:
failures.append(f"outcome_count={len(outcomes)}_expected={request_count}")
if replayed_request_count != request_count:
failures.append(
f"replayed_request_count={replayed_request_count}_expected={request_count}"
)
if expected_request_count is not None and request_count != expected_request_count:
failures.append(
f"materialized_request_count={expected_request_count}_replayed={request_count}"
)
successful = sum(bool(item.get("success")) for item in outcomes if isinstance(item, dict))
tpot_present = sum(item.get("tpot_ms") is not None for item in outcomes if isinstance(item, dict))
ttft_present = sum(item.get("ttft_ms") is not None for item in outcomes if isinstance(item, dict))
nonpositive_tpot = sum(
isinstance(item, dict)
and num(item.get("tpot_ms")) is not None
and num(item.get("tpot_ms")) <= 0
and (nonnegative_int(item.get("completion_tokens")) or 0) > 1
for item in outcomes
)
tokens_verified = all(
isinstance(item, dict)
and item.get("completion_tokens_source") == "usage"
@@ -91,6 +124,8 @@ def read_one(root: Path, k: int, expected_tokens: int | None) -> dict[str, Any]:
failures.append(f"tpot_count={tpot_present}_expected={request_count}")
if ttft_present != request_count:
failures.append(f"ttft_count={ttft_present}_expected={request_count}")
if nonpositive_tpot:
failures.append(f"nonpositive_tpot_for_multi_token_completion={nonpositive_tpot}")
if expected_tokens is not None and not tokens_verified:
failures.append("completion_tokens_not_all_usage_verified")
latency = probe.get("latency_summary") if isinstance(probe.get("latency_summary"), dict) else {}
@@ -101,12 +136,15 @@ def read_one(root: Path, k: int, expected_tokens: int | None) -> dict[str, Any]:
"integrity_failures": failures,
"threshold": num(probe.get("threshold")),
"request_count": request_count,
"replayed_request_count": replayed_request_count,
"materialized_request_count": expected_request_count,
"request_rate": num(probe.get("request_rate")),
"feasible": bool(probe.get("feasible")),
"pass_rate": num(probe.get("pass_rate")),
"success_count": successful,
"ttft_count": ttft_present,
"tpot_count": tpot_present,
"nonpositive_tpot_count": nonpositive_tpot,
"tpot_ms": {name: num(tpot.get(name)) for name in ("mean", "p50", "p90", "p95", "p99")},
}
)
@@ -157,26 +195,57 @@ def data_sanity(records: list[dict[str, Any]]) -> dict[str, Any]:
and record.get("tpot_count") == record.get("request_count")
for record in valid
),
"all_materialized_request_counts_match": all(
record.get("materialized_request_count") is None
or record.get("request_count") == record.get("materialized_request_count")
for record in valid
),
"no_nonpositive_tpot_for_multi_token_completion": all(
record.get("nonpositive_tpot_count", 0) == 0 for record in valid
),
},
}
def materialized_request_count(manifest: dict[str, Any]) -> int | None:
windows_path = manifest.get("trace_windows_path")
window_id = manifest.get("trace_window_id")
if not isinstance(windows_path, str) or not isinstance(window_id, str):
return None
path = Path(windows_path)
if not path.is_file():
return None
payload = load_json(path)
windows = payload.get("windows") if isinstance(payload, dict) else payload
if not isinstance(windows, list):
return None
for window in windows:
if not isinstance(window, dict) or window.get("window_id") != window_id:
continue
value = window.get("num_requests")
if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
return value
return None
def markdown(records: list[dict[str, Any]], sanity: dict[str, Any]) -> str:
lines = [
"# CollectiveSpec fresh-engine fixed-grid summary",
"",
"| configuration | valid | feasible | offered req/s | completed | pass rate | p95 TPOT (ms) | p99 TPOT (ms) | integrity failures |",
"|---|---:|---:|---:|---:|---:|---:|---:|---|",
"| configuration | valid | feasible | offered req/s | materialized/replayed | completed | pass rate | p95 TPOT (ms) | p99 TPOT (ms) | integrity failures |",
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---|",
]
for record in records:
tpot = record.get("tpot_ms") or {}
fmt = lambda value: "—" if value is None else f"{value:.6g}"
lines.append(
"| {label} | {valid} | {feasible} | {rate} | {completed}/{count} | {pass_rate} | {p95} | {p99} | {failures} |".format(
"| {label} | {valid} | {feasible} | {rate} | {materialized}/{replayed} | {completed}/{count} | {pass_rate} | {p95} | {p99} | {failures} |".format(
label=record["label"],
valid=record.get("valid", False),
feasible=record.get("feasible", "—"),
rate=fmt(record.get("request_rate")),
materialized=record.get("materialized_request_count", "—"),
replayed=record.get("replayed_request_count", "—"),
completed=record.get("success_count", "—"),
count=record.get("request_count", "—"),
pass_rate=fmt(record.get("pass_rate")),
@@ -199,8 +268,9 @@ def main() -> int:
expected_tokens = manifest.get("completion_tokens_override")
if isinstance(expected_tokens, bool) or not isinstance(expected_tokens, int):
expected_tokens = None
expected_request_count = materialized_request_count(manifest)
records = [
read_one(args.root, int(k), expected_tokens)
read_one(args.root, int(k), expected_tokens, expected_request_count)
for k in sorted(int(k) for k in manifest["k_values"])
]
sanity = data_sanity(records)