Audit Frontier prefix-cache trace contract
This commit is contained in:
@@ -21,6 +21,11 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--port", type=int, required=True)
|
||||
parser.add_argument("--requests-file", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--served-model",
|
||||
default=None,
|
||||
help="Optional server-side model alias used only for HTTP routing.",
|
||||
)
|
||||
parser.add_argument("--tpot-slo-ms", type=float, default=150.0)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=1800.0)
|
||||
return parser.parse_args()
|
||||
@@ -64,6 +69,7 @@ async def request_one(
|
||||
scheduled_at: float,
|
||||
benchmark_start: float,
|
||||
tpot_slo_ms: float,
|
||||
served_model: str | None,
|
||||
) -> dict[str, Any]:
|
||||
loop = asyncio.get_running_loop()
|
||||
delay = scheduled_at - loop.time()
|
||||
@@ -81,6 +87,8 @@ async def request_one(
|
||||
"success": False,
|
||||
}
|
||||
body = dict(row["body"])
|
||||
if served_model is not None:
|
||||
body["model"] = served_model
|
||||
body.update(
|
||||
{
|
||||
"temperature": 0,
|
||||
@@ -178,6 +186,7 @@ async def replay(args: argparse.Namespace, rows: list[dict[str, Any]]) -> list[d
|
||||
scheduled_at=benchmark_start + float(row["arrived_at"]),
|
||||
benchmark_start=benchmark_start,
|
||||
tpot_slo_ms=args.tpot_slo_ms,
|
||||
served_model=args.served_model,
|
||||
)
|
||||
)
|
||||
for row in rows
|
||||
@@ -219,6 +228,7 @@ def main() -> None:
|
||||
"last_arrival_s": arrivals[-1],
|
||||
"arrival": "original_trace_timestamp_and_order",
|
||||
"input_output_prompt": "exact_source_values",
|
||||
"served_model_alias": args.served_model,
|
||||
"ttft_slo": "1000ms + 1000ms * input_tokens / 8000",
|
||||
"tpot_slo_ms": args.tpot_slo_ms,
|
||||
"target_pass_rate": TARGET_PASS_RATE,
|
||||
|
||||
Reference in New Issue
Block a user