Correct F1 to a steady fixed-QPS workload

This commit is contained in:
2026-07-17 10:36:28 +08:00
parent 10afe4e2e8
commit ec775cdafd
6 changed files with 4004 additions and 2 deletions

View File

@@ -26,6 +26,7 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--input-tokens", type=int, required=True)
parser.add_argument("--output-tokens", type=int, required=True)
parser.add_argument("--requests", type=int, default=64)
parser.add_argument("--duration-seconds", type=float)
parser.add_argument("--rate", type=float, action="append", required=True)
parser.add_argument("--block-size", type=int, default=16)
return parser.parse_args()
@@ -47,6 +48,7 @@ def materialize(
requests: int,
rates: list[float],
block_size: int,
duration_seconds: float | None = None,
) -> dict[str, object]:
values = [input_tokens, output_tokens, requests, block_size]
if any(value <= 0 for value in values):
@@ -57,15 +59,21 @@ def materialize(
raise ValueError("rates must be positive and finite")
if len(set(rates)) != len(rates):
raise ValueError("rates must be unique")
if duration_seconds is not None and duration_seconds <= 0:
raise ValueError("duration must be positive")
root.mkdir(parents=True, exist_ok=True)
anchors = []
blocks = math.ceil(input_tokens / block_size)
for rate in rates:
request_count = max(
requests,
math.ceil(rate * duration_seconds) if duration_seconds is not None else 0,
)
path = root / f"{rate_key(rate)}.csv"
with path.open("w", newline="") as output:
writer = csv.DictWriter(output, fieldnames=FIELDS, lineterminator="\n")
writer.writeheader()
for request_id in range(requests):
for request_id in range(request_count):
first_block = request_id * blocks + 1
writer.writerow(
{
@@ -82,6 +90,7 @@ def materialize(
{
"label": rate_key(rate),
"rate": rate,
"requests": request_count,
"path": path.name,
"sha256": sha256(path),
}
@@ -91,7 +100,8 @@ def materialize(
"contract": {
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"requests": requests,
"minimum_requests_per_anchor": requests,
"minimum_steady_duration_seconds": duration_seconds,
"block_size": block_size,
"arrival": "open_loop_uniform",
"prefix_caching": False,
@@ -114,6 +124,7 @@ def main() -> None:
requests=args.requests,
rates=args.rate,
block_size=args.block_size,
duration_seconds=args.duration_seconds,
)
print(json.dumps(manifest, sort_keys=True))