Report exact state composition verdict
This commit is contained in:
@@ -691,6 +691,16 @@ def analyze(
|
||||
if min(by_batch[config][batch]["n"] for config in CONFIGS) >= 10
|
||||
}
|
||||
observed_real_contrast = REAL_TPOT_MS[CONFIGS[1]] - REAL_TPOT_MS[CONFIGS[0]]
|
||||
exact_iteration_contrast = None
|
||||
if iteration_real is not None:
|
||||
exact_iteration_contrast = (
|
||||
iteration_real[CONFIGS[1]][
|
||||
"decode_token_weighted_iteration_elapsed_ms"
|
||||
]
|
||||
- iteration_real[CONFIGS[0]][
|
||||
"decode_token_weighted_iteration_elapsed_ms"
|
||||
]
|
||||
)
|
||||
decision_contrast = exact_contrast if exact_contrast is not None else proxy_contrast
|
||||
if decision_contrast is None:
|
||||
verdict = "STOP: real Running proxy has insufficient exact simulator support"
|
||||
@@ -706,10 +716,33 @@ def analyze(
|
||||
"to the frozen-real Running proxy, it still predicts TP8 faster."
|
||||
)
|
||||
else:
|
||||
if exact_contrast is not None:
|
||||
verdict = (
|
||||
"Coarse active-batch state can flip Frontier's ordering, but iteration-level "
|
||||
"batch/context telemetry is required before attributing the real gap to state."
|
||||
"Exact real state composition flips Frontier to the correct TP4 winner; "
|
||||
"the ranking error is caused by closed-loop state/composition mismatch, "
|
||||
"not a conditional stage reversal."
|
||||
)
|
||||
else:
|
||||
verdict = (
|
||||
"Coarse active-batch state can flip Frontier's ordering, but "
|
||||
"iteration-level batch/context telemetry is required before attributing "
|
||||
"the real gap to state."
|
||||
)
|
||||
decomposition = None
|
||||
if exact_contrast is not None:
|
||||
decomposition = {
|
||||
"simulator_internal_tp8_minus_tp4_ms": simulator_internal_contrast["total"],
|
||||
"exact_state_matched_tp8_minus_tp4_ms": exact_contrast["total"],
|
||||
"state_composition_shift_ms": (
|
||||
exact_contrast["total"] - simulator_internal_contrast["total"]
|
||||
),
|
||||
"residual_vs_frozen_real_tpot_ms": (
|
||||
exact_contrast["total"] - observed_real_contrast
|
||||
),
|
||||
"residual_vs_exact_iteration_ms": (
|
||||
exact_contrast["total"] - exact_iteration_contrast
|
||||
),
|
||||
}
|
||||
return {
|
||||
"schema": "qwen235-fixed-pd-state-diagnosis-v1",
|
||||
"status": "PASS",
|
||||
@@ -752,9 +785,11 @@ def analyze(
|
||||
if iteration_real is not None
|
||||
else None,
|
||||
"same_batch_contrasts": shared_contrasts,
|
||||
"contrast_decomposition": decomposition,
|
||||
"reference": {
|
||||
"real_tpot_ms": REAL_TPOT_MS,
|
||||
"observed_real_tp8_minus_tp4_ms": observed_real_contrast,
|
||||
"exact_iteration_tp8_minus_tp4_ms": exact_iteration_contrast,
|
||||
},
|
||||
"verdict": verdict,
|
||||
}
|
||||
@@ -776,6 +811,22 @@ def markdown(result: dict[str, Any]) -> str:
|
||||
f"| {result['proxy_matched']['configs'][config]['coverage']:.1%} |"
|
||||
)
|
||||
exact = result["exact_state_matched"]
|
||||
iteration = result["real_iteration_state"]
|
||||
if exact is not None:
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"| Config | Real exact decode batch mean | Real token-weighted iteration ms | Matched simulator ms | Exact coverage |",
|
||||
"|---|---:|---:|---:|---:|",
|
||||
]
|
||||
)
|
||||
for config in CONFIGS:
|
||||
lines.append(
|
||||
f"| {config} | {iteration[config]['decode_batch_size']['mean']:.3f} "
|
||||
f"| {iteration[config]['decode_token_weighted_iteration_elapsed_ms']:.4f} "
|
||||
f"| {exact['configs'][config]['components_ms']['total']:.4f} "
|
||||
f"| {exact['configs'][config]['coverage']:.1%} |"
|
||||
)
|
||||
contrast = (
|
||||
exact["tp8_minus_tp4_ms"]
|
||||
if exact is not None and exact["tp8_minus_tp4_ms"] is not None
|
||||
@@ -824,6 +875,16 @@ def markdown(result: dict[str, Any]) -> str:
|
||||
"## Interpretation boundary",
|
||||
"",
|
||||
f"- Real observed TPOT contrast: {result['reference']['observed_real_tp8_minus_tp4_ms']:+.4f} ms/token.",
|
||||
]
|
||||
)
|
||||
if result["reference"]["exact_iteration_tp8_minus_tp4_ms"] is not None:
|
||||
lines.append(
|
||||
"- Real exact-iteration contrast: "
|
||||
f"{result['reference']['exact_iteration_tp8_minus_tp4_ms']:+.4f} "
|
||||
"ms/decode-token-weighted step."
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
f"- {result['scope']['real_proxy_limitation']}.",
|
||||
f"- {result['scope']['graph_observability']}.",
|
||||
f"- {result['scope']['op_trace_accounting']}.",
|
||||
|
||||
Reference in New Issue
Block a user