144 lines
4.0 KiB
Python
144 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ORACLE_GAP = Path(__file__).resolve().parents[1] / "scripts/oracle_gap"
|
|
sys.path.insert(0, str(ORACLE_GAP))
|
|
|
|
from analyze import score_trial, summarize_trials # noqa: E402
|
|
|
|
|
|
def _request(
|
|
request_id: str,
|
|
admitted: float,
|
|
first: float,
|
|
completed: float,
|
|
*,
|
|
input_tokens: int = 512,
|
|
output_tokens: int = 64,
|
|
) -> dict:
|
|
return {
|
|
"request_id": request_id,
|
|
"scheduled_s": admitted - 0.001,
|
|
"admitted_s": admitted,
|
|
"first_token_s": first,
|
|
"completed_s": completed,
|
|
"input_tokens": input_tokens,
|
|
"requested_output_tokens": output_tokens,
|
|
"actual_output_tokens": output_tokens,
|
|
"success": True,
|
|
"error_kind": None,
|
|
}
|
|
|
|
|
|
def test_score_trial_uses_clean_admission_cohort_and_both_slos(tmp_path: Path) -> None:
|
|
result_path = tmp_path / "result.json"
|
|
result_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"warmup_seconds": 10,
|
|
"clean_segment_seconds": 20,
|
|
"num_clean_segments": 1,
|
|
}
|
|
)
|
|
)
|
|
rows = [
|
|
_request("warmup", 9.0, 9.1, 10.0),
|
|
_request("pass", 11.0, 11.1, 13.0),
|
|
# (16.0 - 12.1) / 63 = 61.9 ms: TPOT miss.
|
|
_request("tpot-fail", 12.0, 12.1, 16.0),
|
|
_request("after-clean", 31.0, 31.1, 32.0),
|
|
]
|
|
request_path = tmp_path / "requests.jsonl"
|
|
request_path.write_text("".join(json.dumps(row) + "\n" for row in rows))
|
|
|
|
scored = score_trial(
|
|
request_path,
|
|
result_path,
|
|
phase="P01",
|
|
config="C00",
|
|
target_rate=0.1,
|
|
repetition=0,
|
|
role="test",
|
|
)
|
|
|
|
assert scored["cohort_n"] == 2
|
|
assert scored["pass_n"] == 1
|
|
assert scored["pass_rate"] == 0.5
|
|
assert scored["slo_goodput_rps"] == 0.05
|
|
assert scored["failure_reasons"] == {"tpot_slo": 1}
|
|
assert scored["offered_rate_valid"]
|
|
assert not scored["feasible"]
|
|
|
|
|
|
def _frontier_row(
|
|
phase: str,
|
|
config: str,
|
|
rate: float,
|
|
feasible: bool,
|
|
) -> dict:
|
|
cohort = 100
|
|
passes = 99 if feasible else 80
|
|
return {
|
|
"phase": phase,
|
|
"config": config,
|
|
"target_rate_rps": rate,
|
|
"repetition": 0,
|
|
"role": "test",
|
|
"cohort_n": cohort,
|
|
"pass_n": passes,
|
|
"pass_rate": passes / cohort,
|
|
"slo_goodput_rps": rate * passes / cohort,
|
|
"feasible": feasible,
|
|
"invariants": {"test": True},
|
|
}
|
|
|
|
|
|
def test_conservative_oracle_bound_can_refute_ten_percent_gate() -> None:
|
|
brackets = {
|
|
"P01": {
|
|
"C00": (30.0, 32.0),
|
|
"C10": (28.0, 30.0),
|
|
"C01": (29.0, 31.0),
|
|
"C11": (27.0, 29.0),
|
|
},
|
|
"P06": {
|
|
"C00": (1.6, 1.7),
|
|
"C10": (1.7, 1.8),
|
|
"C01": (1.55, 1.65),
|
|
"C11": (1.6, 1.7),
|
|
},
|
|
}
|
|
rows = []
|
|
for phase, configs in brackets.items():
|
|
for config, (lower, upper) in configs.items():
|
|
for _ in range(3):
|
|
rows.append(_frontier_row(phase, config, lower, True))
|
|
rows.append(_frontier_row(phase, config, upper, False))
|
|
|
|
summary = summarize_trials(rows)
|
|
|
|
assert summary["verdict"] == "REFUTED"
|
|
assert summary["worst_mixture_conservative"]["gap"] < 0.10
|
|
assert summary["sanity"]["invariants"]["all_cells_bracketed"]
|
|
assert summary["sanity"]["invariants"]["all_frontiers_monotone"]
|
|
|
|
|
|
def test_nonmonotone_frontier_blocks_inference() -> None:
|
|
rows = []
|
|
for phase in ("P01", "P06"):
|
|
for config in ("C00", "C10", "C01", "C11"):
|
|
for _ in range(3):
|
|
rows.append(_frontier_row(phase, config, 1.0, True))
|
|
rows.append(_frontier_row(phase, config, 2.0, False))
|
|
for _ in range(3):
|
|
rows.append(_frontier_row("P01", "C00", 3.0, True))
|
|
|
|
summary = summarize_trials(rows)
|
|
|
|
assert summary["verdict"] == "INCONCLUSIVE"
|
|
assert not summary["sanity"]["invariants"]["all_frontiers_monotone"]
|