Record code-trace canary fidelity gates
This commit is contained in:
@@ -16,7 +16,10 @@ from typing import Any
|
||||
|
||||
|
||||
METRICS = ("ttft", "tpot", "e2e")
|
||||
TPOT_MIN_OUTPUT_TOKENS = (2, 8, 32)
|
||||
SLO_TARGET_PASS_RATE = 0.95
|
||||
PROM_COUNTERS = ("vllm:prefix_cache_queries_total", "vllm:prefix_cache_hits_total")
|
||||
csv.field_size_limit(16 * 1024 * 1024)
|
||||
ITERATION_RE = re.compile(
|
||||
r"Iteration.*?:\s+"
|
||||
r"(?P<context_requests>\d+) context requests, "
|
||||
@@ -165,8 +168,35 @@ def load_sim(root: Path, trace: list[dict[str, str]], trace_sha: str) -> dict[st
|
||||
tail_index = max(range(len(completions)), key=completions.__getitem__)
|
||||
last_arrival = max(float(row["arrived_at"]) for row in trace)
|
||||
summary = json.loads((root / "summary.json").read_text())
|
||||
slo_pass = []
|
||||
for trace_row, metric_row in zip(trace, rows):
|
||||
input_tokens = int(trace_row["num_prefill_tokens"])
|
||||
ttft_threshold_ms = 1000 + 1000 * input_tokens / 8000
|
||||
tpot = (
|
||||
float(metric_row["tpot"])
|
||||
if metric_row["tpot"].strip()
|
||||
else None
|
||||
)
|
||||
slo_pass.append(
|
||||
float(metric_row["ttft"]) <= ttft_threshold_ms
|
||||
and (tpot is None or tpot <= 150)
|
||||
)
|
||||
return {
|
||||
"values": values,
|
||||
"tpot_by_min_output_tokens": {
|
||||
str(threshold): [
|
||||
float(metric_row["tpot"])
|
||||
for trace_row, metric_row in zip(trace, rows)
|
||||
if int(trace_row["num_decode_tokens"]) >= threshold
|
||||
and metric_row["tpot"].strip()
|
||||
]
|
||||
for threshold in TPOT_MIN_OUTPUT_TOKENS
|
||||
},
|
||||
"slo": {
|
||||
"passed": sum(slo_pass),
|
||||
"pass_rate": sum(slo_pass) / len(slo_pass),
|
||||
"feasible": sum(slo_pass) / len(slo_pass) >= SLO_TARGET_PASS_RATE,
|
||||
},
|
||||
"summary": summary,
|
||||
"drain": {
|
||||
"last_arrival_s": last_arrival,
|
||||
@@ -224,6 +254,23 @@ def load_real(
|
||||
"root": str(root),
|
||||
"result_sha256": sha256(result_path),
|
||||
"values": values,
|
||||
"tpot_by_min_output_tokens": {
|
||||
str(threshold): [
|
||||
float(request["tpot_ms"])
|
||||
for request in requests
|
||||
if int(request["requested_output_tokens"]) >= threshold
|
||||
and request.get("tpot_ms") is not None
|
||||
]
|
||||
for threshold in TPOT_MIN_OUTPUT_TOKENS
|
||||
},
|
||||
"slo": {
|
||||
"passed": sum(bool(request["slo_pass"]) for request in requests),
|
||||
"pass_rate": sum(bool(request["slo_pass"]) for request in requests)
|
||||
/ len(requests),
|
||||
"feasible": sum(bool(request["slo_pass"]) for request in requests)
|
||||
/ len(requests)
|
||||
>= SLO_TARGET_PASS_RATE,
|
||||
},
|
||||
"summary": result["summary"],
|
||||
"prefix_cache": prefix_cache_delta(root),
|
||||
"decode_batch": real_decode_batch(root),
|
||||
@@ -262,19 +309,60 @@ def main() -> None:
|
||||
latency = {}
|
||||
for metric in METRICS:
|
||||
real_dist = distribution(pooled[metric])
|
||||
real_per_trial = [
|
||||
distribution(real["values"][metric]) for real in reals
|
||||
]
|
||||
real_reference = {
|
||||
statistic: statistics.fmean(
|
||||
float(trial[statistic]) for trial in real_per_trial
|
||||
)
|
||||
for statistic in ("mean", "p50", "p90", "p95", "p99")
|
||||
}
|
||||
sim_dist = distribution(sim["values"][metric])
|
||||
latency[metric] = {
|
||||
"real": real_dist,
|
||||
"real_trial_statistic_mean": real_reference,
|
||||
"sim": sim_dist,
|
||||
"relative_bias_percent": {
|
||||
statistic: 100
|
||||
* (float(sim_dist[statistic]) - float(real_dist[statistic]))
|
||||
/ float(real_dist[statistic])
|
||||
* (float(sim_dist[statistic]) - real_reference[statistic])
|
||||
/ real_reference[statistic]
|
||||
for statistic in ("mean", "p50", "p90", "p95", "p99")
|
||||
},
|
||||
"real_per_trial": real_per_trial,
|
||||
}
|
||||
tpot_sensitivity = {}
|
||||
for threshold in TPOT_MIN_OUTPUT_TOKENS:
|
||||
key = str(threshold)
|
||||
real_per_trial = [
|
||||
distribution(real["tpot_by_min_output_tokens"][key])
|
||||
for real in reals
|
||||
]
|
||||
real_values = [
|
||||
value
|
||||
for real in reals
|
||||
for value in real["tpot_by_min_output_tokens"][key]
|
||||
]
|
||||
sim_values = sim["tpot_by_min_output_tokens"][key]
|
||||
real_dist = distribution(real_values)
|
||||
real_reference = {
|
||||
statistic: statistics.fmean(
|
||||
float(trial[statistic]) for trial in real_per_trial
|
||||
)
|
||||
for statistic in ("mean", "p50", "p90", "p95", "p99")
|
||||
}
|
||||
sim_dist = distribution(sim_values)
|
||||
tpot_sensitivity[key] = {
|
||||
"real": real_dist,
|
||||
"real_trial_statistic_mean": real_reference,
|
||||
"real_per_trial": real_per_trial,
|
||||
"sim": sim_dist,
|
||||
"relative_bias_percent": {
|
||||
statistic: 100
|
||||
* (float(sim_dist[statistic]) - real_reference[statistic])
|
||||
/ real_reference[statistic]
|
||||
for statistic in ("mean", "p50", "p90", "p95", "p99")
|
||||
},
|
||||
"real_per_trial": [
|
||||
distribution(real["values"][metric]) for real in reals
|
||||
],
|
||||
}
|
||||
payload = {
|
||||
"schema": "frontier-code-trace-canary-analysis-v1",
|
||||
@@ -287,6 +375,20 @@ def main() -> None:
|
||||
"frontier_csv_sha256": trace_sha,
|
||||
},
|
||||
"latency_ms": latency,
|
||||
"tpot_by_min_output_tokens": tpot_sensitivity,
|
||||
"slo": {
|
||||
"definition": {
|
||||
"ttft_ms": "1000 + 1000 * input_tokens / 8000",
|
||||
"tpot_ms": 150,
|
||||
"target_pass_rate": SLO_TARGET_PASS_RATE,
|
||||
},
|
||||
"real_per_trial": [real["slo"] for real in reals],
|
||||
"sim": sim["slo"],
|
||||
"feasibility_flip": any(
|
||||
real["slo"]["feasible"] != sim["slo"]["feasible"]
|
||||
for real in reals
|
||||
),
|
||||
},
|
||||
"prefix_cache": {
|
||||
"real_per_trial": [real["prefix_cache"] for real in reals],
|
||||
"real_hit_ratio_mean": statistics.fmean(
|
||||
|
||||
Reference in New Issue
Block a user