Harden trace replay measurement integrity
This commit is contained in:
@@ -315,7 +315,15 @@ def stream_chat_completion(
|
||||
):
|
||||
tpot_ms = None
|
||||
else:
|
||||
tpot_ms = ((last_token_at - first_token_at) / max(used_tokens - 1, 1)) * 1000.0
|
||||
# A response that delivers all content-bearing chunks at one observed
|
||||
# instant has no measurable inter-token interval. Reporting 0 ms here
|
||||
# would turn an unobservable TPOT into an artificial SLO pass.
|
||||
elapsed_s = last_token_at - first_token_at
|
||||
tpot_ms = (
|
||||
(elapsed_s / max(used_tokens - 1, 1)) * 1000.0
|
||||
if elapsed_s > 0
|
||||
else None
|
||||
)
|
||||
return StreamMetrics(
|
||||
ttft_ms=ttft_ms,
|
||||
tpot_ms=tpot_ms,
|
||||
|
||||
@@ -38,7 +38,10 @@ def binary_search_max_feasible(
|
||||
for _ in range(max_probes):
|
||||
if cur_high - cur_low <= tolerance and best_payload is not None:
|
||||
break
|
||||
threshold = round((cur_low + cur_high) / 2.0, 12)
|
||||
# Thresholds are also trace-selection boundaries. Rounding a midpoint
|
||||
# can cross a real sampling_u value and silently replay a different
|
||||
# workload than the materialized window specifies.
|
||||
threshold = (cur_low + cur_high) / 2.0
|
||||
probe = cache.get(threshold)
|
||||
if probe is None:
|
||||
probe = evaluator(threshold)
|
||||
|
||||
Reference in New Issue
Block a user