Enable gated long context real trace replay
This commit is contained in:
262
runs/frontier-s3-real-v0/qwen30_exact_trace_client.py
Normal file
262
runs/frontier-s3-real-v0/qwen30_exact_trace_client.py
Normal file
@@ -0,0 +1,262 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Replay a private exact-trace anchor without emitting prompt text."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
TARGET_PASS_RATE = 0.95
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--host", default="127.0.0.1")
|
||||||
|
parser.add_argument("--port", type=int, required=True)
|
||||||
|
parser.add_argument("--requests-file", type=Path, required=True)
|
||||||
|
parser.add_argument("--output", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--served-model",
|
||||||
|
default=None,
|
||||||
|
help="Optional server-side model alias used only for HTTP routing.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--tpot-slo-ms", type=float, default=150.0)
|
||||||
|
parser.add_argument("--timeout-seconds", type=float, default=1800.0)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def percentile(values: list[float], fraction: float) -> float | None:
|
||||||
|
if not values:
|
||||||
|
return None
|
||||||
|
ordered = sorted(values)
|
||||||
|
return ordered[math.ceil(fraction * len(ordered)) - 1]
|
||||||
|
|
||||||
|
|
||||||
|
def ttft_slo_ms(input_tokens: int) -> float:
|
||||||
|
return 1000.0 + 1000.0 * input_tokens / 8000.0
|
||||||
|
|
||||||
|
|
||||||
|
def row_vector_sha256(rows: list[dict[str, Any]]) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
for row in rows:
|
||||||
|
digest.update(
|
||||||
|
json.dumps(
|
||||||
|
[
|
||||||
|
row["source_index"],
|
||||||
|
row["arrived_at"],
|
||||||
|
row["input_length"],
|
||||||
|
row["output_length"],
|
||||||
|
row["session_id"],
|
||||||
|
row["runtime_block_ids"],
|
||||||
|
],
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode()
|
||||||
|
)
|
||||||
|
digest.update(b"\n")
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
async def request_one(
|
||||||
|
session: aiohttp.ClientSession,
|
||||||
|
row: dict[str, Any],
|
||||||
|
*,
|
||||||
|
scheduled_at: float,
|
||||||
|
benchmark_start: float,
|
||||||
|
tpot_slo_ms: float,
|
||||||
|
served_model: str | None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
delay = scheduled_at - loop.time()
|
||||||
|
if delay > 0:
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
admitted_at = loop.time()
|
||||||
|
record: dict[str, Any] = {
|
||||||
|
"source_index": int(row["source_index"]),
|
||||||
|
"session_id": int(row["session_id"]),
|
||||||
|
"scheduled_s": scheduled_at - benchmark_start,
|
||||||
|
"admitted_s": admitted_at - benchmark_start,
|
||||||
|
"admission_lag_ms": (admitted_at - scheduled_at) * 1000.0,
|
||||||
|
"input_tokens": int(row["input_length"]),
|
||||||
|
"requested_output_tokens": int(row["output_length"]),
|
||||||
|
"success": False,
|
||||||
|
}
|
||||||
|
body = dict(row["body"])
|
||||||
|
if served_model is not None:
|
||||||
|
body["model"] = served_model
|
||||||
|
body.update(
|
||||||
|
{
|
||||||
|
"temperature": 0,
|
||||||
|
"stream": True,
|
||||||
|
"stream_options": {"include_usage": True},
|
||||||
|
"return_token_ids": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
started = loop.time()
|
||||||
|
async with session.post("/v1/completions", json=body) as response:
|
||||||
|
if response.status != 200:
|
||||||
|
detail = (await response.text())[:1000]
|
||||||
|
raise RuntimeError(f"HTTP {response.status}: {detail}")
|
||||||
|
first_token_at = None
|
||||||
|
last_token_at = None
|
||||||
|
streamed_tokens = 0
|
||||||
|
usage = None
|
||||||
|
while True:
|
||||||
|
raw = await response.content.readline()
|
||||||
|
if not raw:
|
||||||
|
break
|
||||||
|
line = raw.decode(errors="replace").strip()
|
||||||
|
if not line.startswith("data:"):
|
||||||
|
continue
|
||||||
|
data = line[5:].strip()
|
||||||
|
if data == "[DONE]":
|
||||||
|
break
|
||||||
|
payload = json.loads(data)
|
||||||
|
if payload.get("usage"):
|
||||||
|
usage = payload["usage"]
|
||||||
|
emitted = 0
|
||||||
|
for choice in payload.get("choices") or []:
|
||||||
|
token_ids = choice.get("token_ids") or []
|
||||||
|
emitted += len(token_ids) if token_ids else int(bool(choice.get("text")))
|
||||||
|
if emitted:
|
||||||
|
now = loop.time()
|
||||||
|
first_token_at = first_token_at or now
|
||||||
|
last_token_at = now
|
||||||
|
streamed_tokens += emitted
|
||||||
|
finished = loop.time()
|
||||||
|
if first_token_at is None or last_token_at is None or usage is None:
|
||||||
|
raise RuntimeError("missing streaming token or usage")
|
||||||
|
actual_input = int(usage["prompt_tokens"])
|
||||||
|
actual_output = int(usage["completion_tokens"])
|
||||||
|
if actual_input != int(row["input_length"]) or actual_output != int(
|
||||||
|
row["output_length"]
|
||||||
|
):
|
||||||
|
raise RuntimeError(f"usage mismatch: {actual_input}+{actual_output}")
|
||||||
|
ttft = (first_token_at - started) * 1000.0
|
||||||
|
tpot = (
|
||||||
|
(last_token_at - first_token_at) * 1000.0 / (actual_output - 1)
|
||||||
|
if actual_output > 1
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
record.update(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"actual_input_tokens": actual_input,
|
||||||
|
"actual_output_tokens": actual_output,
|
||||||
|
"streamed_token_count": streamed_tokens,
|
||||||
|
"ttft_ms": ttft,
|
||||||
|
"tpot_ms": tpot,
|
||||||
|
"e2e_ms": (finished - started) * 1000.0,
|
||||||
|
"slo_pass": ttft <= ttft_slo_ms(actual_input)
|
||||||
|
and (tpot is None or tpot <= tpot_slo_ms),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception as error:
|
||||||
|
record.update(
|
||||||
|
{
|
||||||
|
"error": f"{type(error).__name__}: {error}",
|
||||||
|
"slo_pass": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return record
|
||||||
|
|
||||||
|
|
||||||
|
async def replay(args: argparse.Namespace, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
|
timeout = aiohttp.ClientTimeout(total=args.timeout_seconds)
|
||||||
|
connector = aiohttp.TCPConnector(limit=0, ttl_dns_cache=300, force_close=True)
|
||||||
|
benchmark_start = asyncio.get_running_loop().time() + 2.0
|
||||||
|
async with aiohttp.ClientSession(
|
||||||
|
base_url=f"http://{args.host}:{args.port}",
|
||||||
|
timeout=timeout,
|
||||||
|
connector=connector,
|
||||||
|
) as session:
|
||||||
|
tasks = [
|
||||||
|
asyncio.create_task(
|
||||||
|
request_one(
|
||||||
|
session,
|
||||||
|
row,
|
||||||
|
scheduled_at=benchmark_start + float(row["arrived_at"]),
|
||||||
|
benchmark_start=benchmark_start,
|
||||||
|
tpot_slo_ms=args.tpot_slo_ms,
|
||||||
|
served_model=args.served_model,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
return await asyncio.gather(*tasks)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
args = parse_args()
|
||||||
|
if args.tpot_slo_ms <= 0 or args.timeout_seconds <= 0:
|
||||||
|
raise ValueError("SLO and timeout must be positive")
|
||||||
|
rows = [json.loads(line) for line in args.requests_file.open() if line.strip()]
|
||||||
|
if not rows:
|
||||||
|
raise ValueError("requests file is empty")
|
||||||
|
arrivals = [float(row["arrived_at"]) for row in rows]
|
||||||
|
if any(right < left for left, right in zip(arrivals, arrivals[1:])):
|
||||||
|
raise ValueError("request arrival order drift")
|
||||||
|
requests = asyncio.run(replay(args, rows))
|
||||||
|
requests.sort(key=lambda row: int(row["source_index"]))
|
||||||
|
completed = [row for row in requests if row["success"]]
|
||||||
|
passed = sum(bool(row["slo_pass"]) for row in requests)
|
||||||
|
ttfts = [float(row["ttft_ms"]) for row in completed]
|
||||||
|
tpots = [
|
||||||
|
float(row["tpot_ms"])
|
||||||
|
for row in completed
|
||||||
|
if row["tpot_ms"] is not None
|
||||||
|
]
|
||||||
|
pass_rate = passed / len(requests)
|
||||||
|
payload = {
|
||||||
|
"schema": "qwen30-exact-trace-anchor-v1",
|
||||||
|
"contract": {
|
||||||
|
"requests_file": str(args.requests_file.resolve()),
|
||||||
|
"requests_file_sha256": hashlib.sha256(
|
||||||
|
args.requests_file.read_bytes()
|
||||||
|
).hexdigest(),
|
||||||
|
"row_vector_sha256": row_vector_sha256(rows),
|
||||||
|
"requests": len(rows),
|
||||||
|
"first_arrival_s": arrivals[0],
|
||||||
|
"last_arrival_s": arrivals[-1],
|
||||||
|
"arrival": "original_trace_timestamp_and_order",
|
||||||
|
"input_output_prompt": "exact_source_values",
|
||||||
|
"served_model_alias": args.served_model,
|
||||||
|
"http_connection_reuse": False,
|
||||||
|
"ttft_slo": "1000ms + 1000ms * input_tokens / 8000",
|
||||||
|
"tpot_slo_ms": args.tpot_slo_ms,
|
||||||
|
"target_pass_rate": TARGET_PASS_RATE,
|
||||||
|
"prompt_text_emitted": False,
|
||||||
|
},
|
||||||
|
"summary": {
|
||||||
|
"completed": len(completed),
|
||||||
|
"failed": len(requests) - len(completed),
|
||||||
|
"passed": passed,
|
||||||
|
"pass_rate": pass_rate,
|
||||||
|
"feasible": pass_rate >= TARGET_PASS_RATE,
|
||||||
|
"ttft_p50_ms": percentile(ttfts, 0.50),
|
||||||
|
"ttft_p95_ms": percentile(ttfts, 0.95),
|
||||||
|
"tpot_p50_ms": percentile(tpots, 0.50),
|
||||||
|
"tpot_p95_ms": percentile(tpots, 0.95),
|
||||||
|
"admission_lag_p95_ms": percentile(
|
||||||
|
[float(row["admission_lag_ms"]) for row in requests], 0.95
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"requests": requests,
|
||||||
|
}
|
||||||
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||||
|
print(json.dumps(payload["summary"], sort_keys=True), flush=True)
|
||||||
|
if len(completed) != len(requests):
|
||||||
|
raise SystemExit(2)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
221
runs/frontier-s3-real-v0/run_full_real.sh
Normal file
221
runs/frontier-s3-real-v0/run_full_real.sh
Normal file
@@ -0,0 +1,221 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
RHO="${1:?usage: run_full_real.sh RHO CONFIG TRIAL PORT}"
|
||||||
|
CONFIG="${2:?usage: run_full_real.sh RHO CONFIG TRIAL PORT}"
|
||||||
|
TRIAL="${3:?usage: run_full_real.sh RHO CONFIG TRIAL PORT}"
|
||||||
|
PORT="${4:?usage: run_full_real.sh RHO CONFIG TRIAL PORT}"
|
||||||
|
CONTROL_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
VENV_ROOT="${VENV_ROOT:-/home/admin/cpfs/wjh/venvs/vllm-0.20.0-cu129-workload-regime-v2}"
|
||||||
|
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
|
||||||
|
SERVED_MODEL="qwen3-30b-s3-real-full"
|
||||||
|
MAX_MODEL_LEN="${MAX_MODEL_LEN:-40960}"
|
||||||
|
ALLOW_SYNTHETIC_PROMPTS="${ALLOW_SYNTHETIC_PROMPTS:-false}"
|
||||||
|
ALLOW_LONG_CONTEXT_SERVER="${ALLOW_LONG_CONTEXT_SERVER:-false}"
|
||||||
|
FLASHINFER_WORKSPACE_BASE="/tmp/wjh/flashinfer-frontier-s3-real-v0-${RHO}-${CONFIG}-t${TRIAL}"
|
||||||
|
SERVER_PID=""
|
||||||
|
TELEMETRY_PID=""
|
||||||
|
|
||||||
|
[[ "${RHO}" =~ ^[A-Za-z0-9._-]+$ ]] || {
|
||||||
|
echo "ERROR: rho label contains unsupported characters: ${RHO}" >&2; exit 1;
|
||||||
|
}
|
||||||
|
if [[ -n "${TRACE_INPUT_ROOT:-}" ]]; then
|
||||||
|
INPUT_NAME=""
|
||||||
|
else
|
||||||
|
case "${RHO}" in
|
||||||
|
0p00125) INPUT_NAME="canary-r0p00125" ;;
|
||||||
|
0p0025) INPUT_NAME="canary-r0p0025" ;;
|
||||||
|
0p005) INPUT_NAME="full-r0p005" ;;
|
||||||
|
0p01) INPUT_NAME="full-r0p01" ;;
|
||||||
|
0p02) INPUT_NAME="full-r0p02" ;;
|
||||||
|
*) echo "ERROR: unsupported built-in rho ${RHO}" >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
# Generic tp<N>_mns<M> parser: TP = tensor-parallel size, MNS = max-num-seqs.
|
||||||
|
if [[ "${CONFIG}" =~ ^tp([0-9]+)_mns([0-9]+)$ ]]; then
|
||||||
|
TP="${BASH_REMATCH[1]}"; MNS="${BASH_REMATCH[2]}"
|
||||||
|
else
|
||||||
|
echo "ERROR: unsupported config ${CONFIG}" >&2; exit 1
|
||||||
|
fi
|
||||||
|
[[ "${TRIAL}" == "1" || "${TRIAL}" == "2" ]] || {
|
||||||
|
echo "ERROR: trial must be 1 or 2" >&2; exit 1;
|
||||||
|
}
|
||||||
|
if (( MAX_MODEL_LEN > 40960 )) && [[ "${ALLOW_LONG_CONTEXT_SERVER}" != "true" ]]; then
|
||||||
|
echo "ERROR: MAX_MODEL_LEN>40960 requires ALLOW_LONG_CONTEXT_SERVER=true" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
TRACE_INPUT_ROOT="${TRACE_INPUT_ROOT:-${CONTROL_ROOT}/inputs/${INPUT_NAME}}"
|
||||||
|
REQUESTS_FILE="${TRACE_INPUT_ROOT}/real_requests.jsonl"
|
||||||
|
INPUT_MANIFEST="${TRACE_INPUT_ROOT}/manifest.json"
|
||||||
|
OUTPUT_ROOT="${OUTPUT_ROOT:-${CONTROL_ROOT}/outputs/full-real/rho-${RHO}/${CONFIG}/trial-${TRIAL}}"
|
||||||
|
|
||||||
|
for path in "${VENV_ROOT}/bin/python" "${VENV_ROOT}/bin/vllm" \
|
||||||
|
"${MODEL_ROOT}/config.json" "${REQUESTS_FILE}" "${INPUT_MANIFEST}" \
|
||||||
|
"${CONTROL_ROOT}/qwen30_exact_trace_client.py" \
|
||||||
|
"${CONTROL_ROOT}/qwen30_prefill_client.py"; do
|
||||||
|
[[ -e "${path}" ]] || { echo "ERROR: missing ${path}" >&2; exit 1; }
|
||||||
|
done
|
||||||
|
[[ ! -e "${OUTPUT_ROOT}" ]] || { echo "ERROR: refusing to overwrite ${OUTPUT_ROOT}" >&2; exit 1; }
|
||||||
|
mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" \
|
||||||
|
"${OUTPUT_ROOT}/results" "${OUTPUT_ROOT}/metrics" \
|
||||||
|
"${OUTPUT_ROOT}/telemetry" "${FLASHINFER_WORKSPACE_BASE}"
|
||||||
|
exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
if [[ -n "${TELEMETRY_PID}" ]] && kill -0 "${TELEMETRY_PID}" 2>/dev/null; then
|
||||||
|
kill "${TELEMETRY_PID}" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
TELEMETRY_PID=""
|
||||||
|
if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||||
|
kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true
|
||||||
|
for _ in $(seq 1 30); do
|
||||||
|
kill -0 "${SERVER_PID}" 2>/dev/null || break
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
SERVER_PID=""
|
||||||
|
}
|
||||||
|
trap cleanup EXIT INT TERM
|
||||||
|
|
||||||
|
IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU allocation is required}"
|
||||||
|
[[ "${#GPU_IDS[@]}" -eq "${TP}" ]] || {
|
||||||
|
echo "ERROR: ${CONFIG} requires exactly ${TP} GPUs" >&2; exit 1;
|
||||||
|
}
|
||||||
|
nvidia-smi --query-gpu=index,name,memory.total,memory.used,utilization.gpu \
|
||||||
|
--format=csv,noheader,nounits > "${OUTPUT_ROOT}/provenance/gpus.before.csv"
|
||||||
|
[[ "$(wc -l < "${OUTPUT_ROOT}/provenance/gpus.before.csv")" -eq 8 ]] || {
|
||||||
|
echo "ERROR: expected eight GPUs on host" >&2; exit 1;
|
||||||
|
}
|
||||||
|
grep -vq 'NVIDIA H20' "${OUTPUT_ROOT}/provenance/gpus.before.csv" && {
|
||||||
|
echo "ERROR: expected eight NVIDIA H20 GPUs" >&2; exit 1;
|
||||||
|
}
|
||||||
|
nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader,nounits \
|
||||||
|
> "${OUTPUT_ROOT}/provenance/compute-apps.before.csv" || true
|
||||||
|
if grep -Eq '^[[:space:]]*[0-9]+' "${OUTPUT_ROOT}/provenance/compute-apps.before.csv"; then
|
||||||
|
echo "ERROR: host is no longer fully idle" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
"${VENV_ROOT}/bin/python" - <<'PY' > "${OUTPUT_ROOT}/provenance/runtime.versions"
|
||||||
|
import torch, transformers, vllm
|
||||||
|
print(f"vllm={vllm.__version__}")
|
||||||
|
print(f"torch={torch.__version__}")
|
||||||
|
print(f"torch_cuda={torch.version.cuda}")
|
||||||
|
print(f"transformers={transformers.__version__}")
|
||||||
|
if vllm.__version__ != "0.20.0" or torch.version.cuda != "12.9":
|
||||||
|
raise SystemExit("runtime version mismatch")
|
||||||
|
PY
|
||||||
|
env | sort > "${OUTPUT_ROOT}/provenance/environment.txt"
|
||||||
|
ps -eo user,pid,ppid,etimes,pcpu,pmem,rss,args --sort=pid \
|
||||||
|
> "${OUTPUT_ROOT}/provenance/processes.before.txt"
|
||||||
|
sha256sum "${REQUESTS_FILE}" "${INPUT_MANIFEST}" \
|
||||||
|
"${CONTROL_ROOT}/qwen30_exact_trace_client.py" \
|
||||||
|
"${CONTROL_ROOT}/qwen30_prefill_client.py" "${BASH_SOURCE[0]}" \
|
||||||
|
> "${OUTPUT_ROOT}/provenance/sources.sha256"
|
||||||
|
INPUT_MANIFEST="${INPUT_MANIFEST}" REQUESTS_FILE="${REQUESTS_FILE}" \
|
||||||
|
MAX_MODEL_LEN="${MAX_MODEL_LEN}" \
|
||||||
|
ALLOW_SYNTHETIC_PROMPTS="${ALLOW_SYNTHETIC_PROMPTS}" \
|
||||||
|
"${VENV_ROOT}/bin/python" - <<'PY'
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
manifest = json.loads(Path(os.environ["INPUT_MANIFEST"]).read_text())
|
||||||
|
source_block_size = int(manifest["source_block_size"])
|
||||||
|
if source_block_size <= 0 or source_block_size % 16 or manifest["target_block_size"] != 16:
|
||||||
|
raise SystemExit("source-to-16 block-size contract failed")
|
||||||
|
failures = {
|
||||||
|
key: value for key, value in manifest["block_contract"].items()
|
||||||
|
if key.endswith(("mismatches", "collisions", "conflicts"))
|
||||||
|
}
|
||||||
|
if any(failures.values()):
|
||||||
|
raise SystemExit(f"block/hash contract failed: {failures}")
|
||||||
|
if (
|
||||||
|
manifest["prompt_contract"]["synthetic_fallback_requests"]
|
||||||
|
and os.environ["ALLOW_SYNTHETIC_PROMPTS"].lower() != "true"
|
||||||
|
):
|
||||||
|
raise SystemExit("full real requires real prompt text for every request")
|
||||||
|
with Path(os.environ["REQUESTS_FILE"]).open() as stream:
|
||||||
|
max_request_tokens = max(
|
||||||
|
int(row["input_length"]) + int(row["output_length"])
|
||||||
|
for row in map(json.loads, stream)
|
||||||
|
)
|
||||||
|
if max_request_tokens > int(os.environ["MAX_MODEL_LEN"]):
|
||||||
|
raise SystemExit(
|
||||||
|
f"trace max request tokens {max_request_tokens} exceeds "
|
||||||
|
f"MAX_MODEL_LEN={os.environ['MAX_MODEL_LEN']}"
|
||||||
|
)
|
||||||
|
PY
|
||||||
|
|
||||||
|
export TOKENIZERS_PARALLELISM=false VLLM_USE_V1=1 TORCH_CUDA_ARCH_LIST=9.0 PREFIX_CACHING=true
|
||||||
|
export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 FLASHINFER_WORKSPACE_BASE
|
||||||
|
if [[ "${ALLOW_LONG_CONTEXT_SERVER}" == "true" ]]; then
|
||||||
|
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||||
|
fi
|
||||||
|
ulimit -n 65536
|
||||||
|
setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
|
||||||
|
--host 127.0.0.1 --port "${PORT}" --served-model-name "${SERVED_MODEL}" \
|
||||||
|
--tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \
|
||||||
|
--max-model-len "${MAX_MODEL_LEN}" --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \
|
||||||
|
--enable-prefix-caching --block-size 16 --enable-chunked-prefill --no-enable-log-requests \
|
||||||
|
--enable-logging-iteration-details \
|
||||||
|
> "${OUTPUT_ROOT}/logs/server.log" 2>&1 &
|
||||||
|
SERVER_PID=$!
|
||||||
|
READY=0
|
||||||
|
for _ in $(seq 1 900); do
|
||||||
|
if curl -fsS --max-time 2 "http://127.0.0.1:${PORT}/v1/models" \
|
||||||
|
> "${OUTPUT_ROOT}/results/models.json" 2>/dev/null; then READY=1; break; fi
|
||||||
|
kill -0 "${SERVER_PID}" 2>/dev/null || { tail -200 "${OUTPUT_ROOT}/logs/server.log"; exit 1; }
|
||||||
|
sleep 3
|
||||||
|
done
|
||||||
|
[[ "${READY}" -eq 1 ]] || { echo "ERROR: server readiness timeout" >&2; exit 1; }
|
||||||
|
|
||||||
|
"${VENV_ROOT}/bin/python" "${CONTROL_ROOT}/qwen30_prefill_client.py" \
|
||||||
|
--port "${PORT}" --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" \
|
||||||
|
--rate 1 --requests 4 --input-tokens 512 --output-tokens 1 \
|
||||||
|
--output "${OUTPUT_ROOT}/results/warmup.json"
|
||||||
|
curl -fsS "http://127.0.0.1:${PORT}/metrics" > "${OUTPUT_ROOT}/metrics/before.prom"
|
||||||
|
(
|
||||||
|
while true; do
|
||||||
|
ts=$(date -u +%s.%N)
|
||||||
|
snap=$(curl -fsS "http://127.0.0.1:${PORT}/metrics" 2>/dev/null \
|
||||||
|
| grep -E 'vllm:(num_requests_running|num_requests_waiting|prefix_cache_hits|prefix_cache_queries|gpu_prefix_cache_hit_rate|gpu_cache_usage)' \
|
||||||
|
| tr '\n' '|')
|
||||||
|
printf '{"wall_time_epoch_s": %s, "metrics": "%s"}\n' "${ts}" "${snap}" \
|
||||||
|
>> "${OUTPUT_ROOT}/telemetry/server-state.jsonl"
|
||||||
|
sleep 15
|
||||||
|
done
|
||||||
|
) &
|
||||||
|
TELEMETRY_PID=$!
|
||||||
|
"${VENV_ROOT}/bin/python" "${CONTROL_ROOT}/qwen30_exact_trace_client.py" \
|
||||||
|
--port "${PORT}" --requests-file "${REQUESTS_FILE}" --served-model "${SERVED_MODEL}" \
|
||||||
|
--output "${OUTPUT_ROOT}/results/result.json" --tpot-slo-ms 150 \
|
||||||
|
--timeout-seconds 5400
|
||||||
|
kill "${TELEMETRY_PID}" 2>/dev/null || true
|
||||||
|
TELEMETRY_PID=""
|
||||||
|
curl -fsS "http://127.0.0.1:${PORT}/metrics" > "${OUTPUT_ROOT}/metrics/after.prom"
|
||||||
|
INPUT_MANIFEST="${INPUT_MANIFEST}" OUTPUT_ROOT="${OUTPUT_ROOT}" \
|
||||||
|
"${VENV_ROOT}/bin/python" - <<'PY'
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
manifest = json.loads(Path(os.environ["INPUT_MANIFEST"]).read_text())
|
||||||
|
result = json.loads((Path(os.environ["OUTPUT_ROOT"]) / "results/result.json").read_text())
|
||||||
|
if result["contract"]["row_vector_sha256"] != manifest["paired_row_vector_sha256"]:
|
||||||
|
raise SystemExit("real/sim paired row-vector digest mismatch")
|
||||||
|
if result["contract"]["requests"] != manifest["requests"]:
|
||||||
|
raise SystemExit("real/sim request-count mismatch")
|
||||||
|
if not all(request["success"] for request in result["requests"]):
|
||||||
|
raise SystemExit("one or more requests failed")
|
||||||
|
PY
|
||||||
|
|
||||||
|
cleanup
|
||||||
|
nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total,memory.used,utilization.gpu \
|
||||||
|
--format=csv,noheader,nounits > "${OUTPUT_ROOT}/provenance/gpus.after.csv"
|
||||||
|
find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \
|
||||||
|
| sort -z | xargs -0 sha256sum > "${OUTPUT_ROOT}/provenance/artifacts.sha256"
|
||||||
|
echo "FULL_REAL_COMPLETE rho=${RHO} config=${CONFIG} trial=${TRIAL}"
|
||||||
Reference in New Issue
Block a user