Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)
Workload-conditioned operator profiling on patched vLLM 0.24.0 + Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64 raggedness, 8-45% token-efficiency loss vs rectangular controls); mechanism decomposition kills the padding narrative and finds the arrival-uniformization artifact (-12.9%); cross-version churn surface shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held. Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer sidecars and metrics are tracked. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
466
runs/opprof-phase6/analyze_phase6.py
Normal file
466
runs/opprof-phase6/analyze_phase6.py
Normal file
@@ -0,0 +1,466 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frozen Phase-6 paired-anchor, frontier, rank, and Layer-1 analysis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
OLD_BEST = "tp2_mns32"
|
||||
TOP20 = {"tp2_mns32", "tp2_mns64", "tp4_mns16", "tp4_mns32", "tp4_mns64"}
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def numeric(values: list[float | int | None]) -> dict[str, Any]:
|
||||
finite = [float(x) for x in values if x is not None and math.isfinite(float(x))]
|
||||
return {
|
||||
"n": len(values), "finite_n": len(finite), "missing_n": len(values) - len(finite),
|
||||
"min": min(finite) if finite else None, "max": max(finite) if finite else None,
|
||||
"distinct_n": len(set(finite)),
|
||||
}
|
||||
|
||||
|
||||
def cv(values: list[float]) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
array = np.asarray(values, dtype=np.float64)
|
||||
mean = float(array.mean())
|
||||
return float(array.std(ddof=0) / mean) if mean else 0.0
|
||||
|
||||
|
||||
def floor_buckets(scores: dict[str, float]) -> tuple[float, dict[str, int]]:
|
||||
tol = max(1e-9, 1e-6 * max(abs(x) for x in scores.values()))
|
||||
return tol, {key: math.floor(value / tol) for key, value in scores.items()}
|
||||
|
||||
|
||||
def kendall_tau_b(x: dict[str, int], y: dict[str, int]) -> dict[str, Any]:
|
||||
keys = sorted(x)
|
||||
c = d = tx = ty = joint = 0
|
||||
for i, left in enumerate(keys):
|
||||
for right in keys[i + 1:]:
|
||||
dx = (x[left] > x[right]) - (x[left] < x[right])
|
||||
dy = (y[left] > y[right]) - (y[left] < y[right])
|
||||
if dx == 0 and dy == 0:
|
||||
joint += 1
|
||||
elif dx == 0:
|
||||
tx += 1
|
||||
elif dy == 0:
|
||||
ty += 1
|
||||
elif dx == dy:
|
||||
c += 1
|
||||
else:
|
||||
d += 1
|
||||
denom = math.sqrt((c + d + tx) * (c + d + ty))
|
||||
return {
|
||||
"tau_b": (c - d) / denom if denom else None, "concordant": c,
|
||||
"discordant": d, "v20_only_ties": tx, "v24_only_ties": ty,
|
||||
"joint_ties": joint, "pairs": len(keys) * (len(keys) - 1) // 2,
|
||||
}
|
||||
|
||||
|
||||
def layer_metrics(records: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
model = [x for x in records if x.get("model_executed")]
|
||||
mode = Counter(str(x["cudagraph"]["runtime_mode"]) for x in model)
|
||||
total_bucket = sum(int(x["cudagraph"]["bucket_tokens"]) for x in model)
|
||||
total_padding = sum(int(x["cudagraph"]["padding_tokens"]) for x in model)
|
||||
kv = [float(x["kv"]["usage"]) for x in model]
|
||||
waiting = [float(x["queues"]["waiting"]) for x in model]
|
||||
decode_b = [float(x["decode_batch_size"]) for x in model]
|
||||
return {
|
||||
"steps": len(records), "model_steps": len(model),
|
||||
"prefill_only_steps": sum(int(x["prefill_tokens"]) > 0 and int(x["decode_tokens"]) == 0 for x in model),
|
||||
"decode_only_steps": sum(int(x["prefill_tokens"]) == 0 and int(x["decode_tokens"]) > 0 for x in model),
|
||||
"mixed_steps": sum(int(x["prefill_tokens"]) > 0 and int(x["decode_tokens"]) > 0 for x in model),
|
||||
"prefill_tokens": sum(int(x["prefill_tokens"]) for x in model),
|
||||
"decode_tokens": sum(int(x["decode_tokens"]) for x in model),
|
||||
"preemptions": sum(int(x["preemptions"]) for x in model),
|
||||
"kv_usage_mean": float(np.mean(kv)) if kv else None,
|
||||
"kv_usage_max": max(kv) if kv else None,
|
||||
"waiting_mean": float(np.mean(waiting)) if waiting else None,
|
||||
"waiting_max": max(waiting) if waiting else None,
|
||||
"waiting_cv": cv(waiting), "decode_B_mean": float(np.mean(decode_b)) if decode_b else None,
|
||||
"decode_B_cv": cv(decode_b), "graph_mode_counts": dict(sorted(mode.items())),
|
||||
"graph_mode_shares": {key: value / len(model) for key, value in sorted(mode.items())} if model else {},
|
||||
"padding_fraction": total_padding / total_bucket if total_bucket else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def load_stream(path: Path) -> list[dict[str, Any]]:
|
||||
return [json.loads(line) for line in path.read_text().splitlines() if "step_index" in json.loads(line)]
|
||||
|
||||
|
||||
def p3_reference(p3_root: Path) -> dict[str, Any]:
|
||||
run = p3_root / "primary/P10-C00/moderate"
|
||||
result = json.loads((run / "client/result.json").read_text())
|
||||
t0 = int(result["t0_mono_ns"])
|
||||
lo = t0 + int(float(result["clean"]["start_s"]) * 1e9)
|
||||
hi = t0 + int(float(result["clean"]["end_s"]) * 1e9)
|
||||
stream = next((run / "opprof").glob("*.jsonl"))
|
||||
records = [x for x in load_stream(stream) if lo <= int(x["submit_mono_ns"]) < hi]
|
||||
return {
|
||||
"limitation": "P3 P10 reference is TP1/MNS-default, steady arrival, sampling_u<=0.125, output<=256; it is contextual rather than a matched v0.20 composition baseline.",
|
||||
"stream_sha256": sha256_file(stream), "metrics": layer_metrics(records),
|
||||
}
|
||||
|
||||
|
||||
def accepted_anchor(anchor_dir: Path) -> dict[str, Any]:
|
||||
primary = json.loads((anchor_dir / "result.json").read_text())
|
||||
cell_dir = anchor_dir.parent
|
||||
anchor = float(primary["anchor"])
|
||||
confirms = []
|
||||
for path in sorted(cell_dir.glob("confirm-*-anchor-*/result.json")):
|
||||
item = json.loads(path.read_text())
|
||||
if math.isclose(float(item["anchor"]), anchor, rel_tol=0, abs_tol=1e-15):
|
||||
confirms.append(item)
|
||||
trials = [primary, *confirms]
|
||||
votes = [bool(x["feasible"]) for x in trials]
|
||||
if len(votes) == 1 or len(set(votes)) == 1:
|
||||
verdict = votes[0]
|
||||
resolved = True
|
||||
elif len(votes) >= 3:
|
||||
verdict = sum(votes) >= 2
|
||||
resolved = True
|
||||
else:
|
||||
verdict = None
|
||||
resolved = False
|
||||
return {
|
||||
"anchor": anchor, "primary": primary, "confirmations": confirms,
|
||||
"trial_count": len(trials), "pass_rates": [x["pass_rate"] for x in trials],
|
||||
"feasible_votes": votes, "accepted_feasible": verdict, "resolved": resolved,
|
||||
"accepted_pass_rate": float(np.median([x["pass_rate"] for x in trials])),
|
||||
}
|
||||
|
||||
|
||||
def matching_anchor_dir(cell_dir: Path, anchor: float) -> Path | None:
|
||||
for path in cell_dir.glob("anchor-*/result.json"):
|
||||
item = json.loads(path.read_text())
|
||||
if math.isclose(float(item["anchor"]), anchor, rel_tol=0, abs_tol=1e-15):
|
||||
return path.parent
|
||||
return None
|
||||
|
||||
|
||||
def analyze(root: Path, ground_path: Path, p3_root: Path) -> dict[str, Any]:
|
||||
ground = json.loads(ground_path.read_text())
|
||||
old_cells = {x["cell_id"]: x for x in ground["cells"]}
|
||||
old_probe = {
|
||||
(x["cell_id"], float(p["sampling_u"])): p
|
||||
for x in ground["cells"] for p in x["probe_history"]
|
||||
}
|
||||
cells: dict[str, Any] = {}
|
||||
all_primary = []
|
||||
all_client_invariants = []
|
||||
selection_matches = []
|
||||
for cell, old in old_cells.items():
|
||||
cell_dir = root / "solo-authoritative/cells" / cell
|
||||
colocated_dir = root / "cells" / cell
|
||||
f20 = float(old["measured_objective"]["slo_feasible_req_s_per_gpu"])
|
||||
if not (cell_dir / "cell-valid.json").exists():
|
||||
cells[cell] = {
|
||||
"tp": old["tensor_parallel_size"], "mns": old["max_num_seqs"],
|
||||
"measurement_status": "UNMEASURED_SOLO",
|
||||
"f20": f20, "f24": None, "drift": None,
|
||||
"observed_max_feasible_rate": None,
|
||||
"material_frontier_moved": None, "boundary": "UNMEASURED",
|
||||
"bounded": False, "censor": "UNMEASURED_SOLO",
|
||||
"anchors": [], "cell_valid": None,
|
||||
}
|
||||
continue
|
||||
valid = json.loads((cell_dir / "cell-valid.json").read_text())
|
||||
stream = next((cell_dir / "opprof").glob("*.jsonl"))
|
||||
stream_records = load_stream(stream)
|
||||
anchors = []
|
||||
for path in sorted(cell_dir.glob("anchor-*/result.json")):
|
||||
accepted = accepted_anchor(path.parent)
|
||||
primary = accepted["primary"]
|
||||
key = (cell, float(primary["anchor"]))
|
||||
historical = old_probe[key]
|
||||
lo = int(primary["interval"]["start_mono_ns"])
|
||||
hi = int(primary["interval"]["end_mono_ns"])
|
||||
records = [x for x in stream_records if lo <= int(x["submit_mono_ns"]) <= hi]
|
||||
accepted["layer1"] = layer_metrics(records)
|
||||
for confirmation in accepted["confirmations"]:
|
||||
confirm_lo = int(confirmation["interval"]["start_mono_ns"])
|
||||
confirm_hi = int(confirmation["interval"]["end_mono_ns"])
|
||||
confirmation["layer1"] = layer_metrics([
|
||||
x for x in stream_records
|
||||
if confirm_lo <= int(x["submit_mono_ns"]) <= confirm_hi
|
||||
])
|
||||
accepted["v20"] = {
|
||||
"pass_rate": historical["pass_rate"], "feasible": historical["feasible"],
|
||||
"request_count": historical["request_count"],
|
||||
"rate_per_gpu": historical["request_rate_per_gpu_req_s_gpu"],
|
||||
}
|
||||
accepted["v24"] = {
|
||||
"pass_rate": accepted["accepted_pass_rate"],
|
||||
"feasible": accepted["accepted_feasible"],
|
||||
"request_count": primary["selection"]["count"],
|
||||
"rate_per_gpu": primary["selection"]["offered_req_s_per_gpu"],
|
||||
}
|
||||
colocated_anchor_dir = matching_anchor_dir(colocated_dir, float(primary["anchor"]))
|
||||
if colocated_anchor_dir is not None:
|
||||
colocated = accepted_anchor(colocated_anchor_dir)
|
||||
accepted["colocated"] = {
|
||||
"primary_pass_rate": colocated["primary"]["pass_rate"],
|
||||
"primary_feasible": colocated["primary"]["feasible"],
|
||||
"trial_pass_rates": colocated["pass_rates"],
|
||||
"accepted_feasible": colocated["accepted_feasible"],
|
||||
"solo_minus_colocated_primary_pass_rate": (
|
||||
accepted["accepted_pass_rate"] - colocated["primary"]["pass_rate"]
|
||||
),
|
||||
}
|
||||
else:
|
||||
accepted["colocated"] = None
|
||||
accepted["feasibility_flip"] = (
|
||||
accepted["accepted_feasible"] is not None
|
||||
and accepted["accepted_feasible"] != historical["feasible"]
|
||||
)
|
||||
anchors.append(accepted)
|
||||
all_primary.append(primary)
|
||||
all_client_invariants.extend(primary["invariants"].values())
|
||||
selection_matches.append(primary["selection"]["count"] == historical["request_count"])
|
||||
peak_u = float(old["measured_objective"]["best_sampling_u"])
|
||||
peak = next(x for x in anchors if math.isclose(x["anchor"], peak_u, abs_tol=1e-15))
|
||||
feasible = [x for x in anchors if x["accepted_feasible"] is True]
|
||||
infeasible = [x for x in anchors if x["accepted_feasible"] is False]
|
||||
unresolved = [x for x in anchors if x["accepted_feasible"] is None]
|
||||
observed_frontier = max((x["v24"]["rate_per_gpu"] for x in feasible), default=None)
|
||||
lower = [x for x in anchors if x["anchor"] < peak_u]
|
||||
upper = [x for x in anchors if x["anchor"] > peak_u]
|
||||
if unresolved:
|
||||
bounded, censor = False, "UNRESOLVED_SOLO_ANCHOR"
|
||||
elif feasible and infeasible:
|
||||
monotonic = max(x["anchor"] for x in feasible) < min(x["anchor"] for x in infeasible)
|
||||
bounded = monotonic
|
||||
censor = None if bounded else "NONMONOTONIC_SOLO_ANCHORS"
|
||||
elif feasible:
|
||||
bounded, censor = False, "RIGHT_CENSORED_HISTORY_EDGE"
|
||||
elif infeasible:
|
||||
bounded, censor = False, "LEFT_CENSORED_HISTORY_EDGE"
|
||||
else:
|
||||
bounded, censor = False, "NO_RESOLVED_SOLO_ANCHOR"
|
||||
frontier = (
|
||||
None if censor in {
|
||||
"UNRESOLVED_SOLO_ANCHOR", "NONMONOTONIC_SOLO_ANCHORS",
|
||||
"NO_RESOLVED_SOLO_ANCHOR",
|
||||
} else observed_frontier
|
||||
)
|
||||
drift = (frontier / f20 - 1) if frontier is not None else None
|
||||
if censor == "NONMONOTONIC_SOLO_ANCHORS":
|
||||
boundary = "NONMONOTONIC"
|
||||
elif peak["accepted_feasible"] is False:
|
||||
boundary = "DOWN"
|
||||
elif peak["accepted_feasible"] is None:
|
||||
boundary = "UNRESOLVED"
|
||||
elif any(x["accepted_feasible"] is True for x in upper):
|
||||
boundary = "UP"
|
||||
else:
|
||||
boundary = "STABLE"
|
||||
cells[cell] = {
|
||||
"tp": old["tensor_parallel_size"], "mns": old["max_num_seqs"],
|
||||
"measurement_status": "SOLO_AUTHORITATIVE",
|
||||
"f20": f20, "f24": frontier, "drift": drift,
|
||||
"observed_max_feasible_rate": observed_frontier,
|
||||
"material_frontier_moved": bounded and drift is not None and abs(drift) > .05,
|
||||
"boundary": boundary, "bounded": bounded, "censor": censor,
|
||||
"anchors": anchors, "cell_valid": valid,
|
||||
}
|
||||
|
||||
scores20 = {key: value["f20"] for key, value in cells.items()}
|
||||
scores24 = {key: value["f24"] for key, value in cells.items() if value["f24"] is not None}
|
||||
tol20, buckets20 = floor_buckets(scores20)
|
||||
tol24, buckets24 = floor_buckets(scores24)
|
||||
full_bounded = len(scores24) == 12 and all(x["bounded"] for x in cells.values())
|
||||
max24 = max(buckets24.values())
|
||||
if not full_bounded:
|
||||
argmax = "INCONCLUSIVE"
|
||||
else:
|
||||
argmax = "SURVIVED" if buckets24[OLD_BEST] == max24 else "MOVED"
|
||||
tau = kendall_tau_b(buckets20, buckets24) if full_bounded else None
|
||||
reversals = []
|
||||
for left in sorted(TOP20):
|
||||
for right in sorted(TOP20):
|
||||
if left >= right:
|
||||
continue
|
||||
if not cells[left]["bounded"] or not cells[right]["bounded"]:
|
||||
continue
|
||||
if left not in scores24 or right not in scores24:
|
||||
continue
|
||||
old_delta = scores20[left] - scores20[right]
|
||||
if abs(old_delta) / max(scores20[left], scores20[right]) <= .05:
|
||||
continue
|
||||
new_delta = scores24[left] - scores24[right]
|
||||
if old_delta * new_delta < 0:
|
||||
reversals.append([left, right])
|
||||
if not full_bounded or argmax == "INCONCLUSIVE" or tau is None:
|
||||
ranking = "INCONCLUSIVE"
|
||||
elif argmax == "MOVED" or tau["tau_b"] < .8 or reversals:
|
||||
ranking = "MOVED"
|
||||
else:
|
||||
ranking = "SURVIVED"
|
||||
trap_inputs = ["tp4_mns8", "tp4_mns16", "tp4_mns32"]
|
||||
if not all(cells[x]["bounded"] for x in trap_inputs):
|
||||
trap = "INCONCLUSIVE"
|
||||
elif buckets24["tp4_mns16"] == max24:
|
||||
trap = "CEASES_TO_BE_A_TRAP"
|
||||
elif buckets24["tp4_mns16"] >= max(buckets24["tp4_mns8"], buckets24["tp4_mns32"]):
|
||||
trap = "PERSISTS"
|
||||
else:
|
||||
trap = "ESCAPES"
|
||||
|
||||
colocated_state = json.loads((root / "controller-state.json").read_text())
|
||||
solo_state_path = root / "solo-authoritative/controller-state.json"
|
||||
solo_state = json.loads(solo_state_path.read_text()) if solo_state_path.exists() else None
|
||||
state = solo_state or colocated_state
|
||||
measured_cells = [
|
||||
x for x in cells.values() if x["measurement_status"] == "SOLO_AUTHORITATIVE"
|
||||
]
|
||||
coverage = {
|
||||
"solo_primary_anchors_at_least_25": len(all_primary) >= 25,
|
||||
"solo_measured_cells_12": len(measured_cells) == 12,
|
||||
}
|
||||
invariants = {
|
||||
"surface_rows_12": len(cells) == 12,
|
||||
"selection_counts_match": all(selection_matches),
|
||||
"client_invariants": all(all_client_invariants),
|
||||
"cell_validity": all(
|
||||
all(x["cell_valid"]["invariants"].values()) for x in measured_cells
|
||||
),
|
||||
"gpu_below_6": float(state["gpu_hours_total"]) < 6.0,
|
||||
"rates_nonnegative": all(x["f24"] is None or x["f24"] >= 0 for x in cells.values()),
|
||||
"surface_not_identical": len(set(scores24.values())) > 1,
|
||||
}
|
||||
red_flags = [key for key, value in {**coverage, **invariants}.items() if not value]
|
||||
drifted = [key for key, value in cells.items() if value["material_frontier_moved"] is True]
|
||||
flip_cells = [
|
||||
key for key, value in cells.items()
|
||||
if any(anchor["feasibility_flip"] for anchor in value["anchors"])
|
||||
]
|
||||
partial = bool([key for key, value in coverage.items() if not value])
|
||||
w1_audit_path = root / "w1-readjudication-A-P6-1.json"
|
||||
w1_audit = json.loads(w1_audit_path.read_text()) if w1_audit_path.exists() else None
|
||||
censored = {key: value["censor"] for key, value in cells.items() if not value["bounded"]}
|
||||
colocated_deltas = [
|
||||
{
|
||||
"cell": cell, "anchor": anchor["anchor"],
|
||||
"solo_pass_rate": anchor["accepted_pass_rate"],
|
||||
"solo_feasible": anchor["accepted_feasible"],
|
||||
**anchor["colocated"],
|
||||
}
|
||||
for cell, value in cells.items() for anchor in value["anchors"]
|
||||
if anchor.get("colocated") is not None
|
||||
]
|
||||
return {
|
||||
"schema": 1,
|
||||
"status": "BUDGET_STOP_PARTIAL" if partial else ("VALID" if not red_flags else "INVALID"),
|
||||
"authoritative_tier": "A-P6-2 solo host placement",
|
||||
"limitation": "Upgrade-path churn includes dash1->dash0 and resolved-default changes. Co-located W1-W3 values are indicative only; P3 composition is contextual, not a matched v0.20 Layer-1 baseline.",
|
||||
"ground_truth_sha256": sha256_file(ground_path), "cells": cells,
|
||||
"floor_buckets": {"v20_tol": tol20, "v20": buckets20, "v24_tol": tol24, "v24": buckets24},
|
||||
"verdicts": {
|
||||
"argmax": argmax, "ranking": ranking, "trap": trap,
|
||||
"full_surface_bounded": full_bounded, "tau_b": tau,
|
||||
"top_pair_reversals_gt5pct": reversals,
|
||||
},
|
||||
"materially_drifted_cells": drifted,
|
||||
"feasibility_flip_cells": flip_cells,
|
||||
"decision_blockers": {
|
||||
"coverage": [key for key, value in coverage.items() if not value],
|
||||
"unbounded_or_unresolved_cells": censored,
|
||||
},
|
||||
"solo_vs_colocated": colocated_deltas,
|
||||
"w1_readjudication": w1_audit,
|
||||
"run_stats": {
|
||||
"measured_cells": len(measured_cells),
|
||||
"surface_cells": len(cells),
|
||||
"primary_anchor_runs": len(all_primary),
|
||||
"confirmation_runs": sum(
|
||||
len(anchor["confirmations"])
|
||||
for cell in cells.values() for anchor in cell["anchors"]
|
||||
),
|
||||
"accepted_anchor_trials": sum(
|
||||
anchor["trial_count"]
|
||||
for cell in cells.values() for anchor in cell["anchors"]
|
||||
),
|
||||
"warmup_runs": len(measured_cells),
|
||||
"solo_cell_gpu_hours": {
|
||||
key: value.get("gpu_hours") for key, value in (solo_state or {}).get("cells", {}).items()
|
||||
},
|
||||
"colocated_wave_gpu_hours": {
|
||||
key: value.get("gpu_hours") for key, value in colocated_state["waves"].items()
|
||||
},
|
||||
"launch_echo": (
|
||||
(root / "launch-echo.log").read_text().splitlines()
|
||||
+ ((root / "solo-authoritative/launch-echo.log").read_text().splitlines()
|
||||
if (root / "solo-authoritative/launch-echo.log").exists() else [])
|
||||
),
|
||||
},
|
||||
"attempt_history": {
|
||||
"colocated_status": colocated_state["status"],
|
||||
"colocated_h20_hours": colocated_state["gpu_hours_total"],
|
||||
"colocated_primary_anchors": colocated_state["completed_primary_anchors"],
|
||||
"colocated_confirmations": colocated_state["completed_confirmations"],
|
||||
"colocated_budget_stop": colocated_state.get("budget_stop"),
|
||||
"solo_status": (solo_state or {}).get("status"),
|
||||
"solo_repairs": (solo_state or {}).get("repairs", []),
|
||||
"solo_failures": (solo_state or {}).get("failures", []),
|
||||
"raw_roots": {
|
||||
"colocated": str(root / "cells"),
|
||||
"solo_authoritative": str(root / "solo-authoritative/cells"),
|
||||
},
|
||||
},
|
||||
"p3_composition_reference": p3_reference(p3_root),
|
||||
"gpu": {
|
||||
"new_h20_hours": state["gpu_hours_total"], "hard_cap": 6.0,
|
||||
"prior_colocated_h20_hours": colocated_state["gpu_hours_total"],
|
||||
"solo_h20_hours": (solo_state or {}).get("solo_gpu_hours", 0.0),
|
||||
"completed_primary_anchors": (solo_state or {}).get("primary_anchors", 0),
|
||||
"confirmations": (solo_state or {}).get("confirmations", 0),
|
||||
"controller_status": state["status"],
|
||||
"budget_stop": state.get("budget_stop"),
|
||||
},
|
||||
"sanity": {
|
||||
"red_flags": red_flags, "coverage": coverage, "invariants": invariants,
|
||||
"numeric": {
|
||||
"v20_score": numeric(list(scores20.values())),
|
||||
"v24_score": numeric(list(scores24.values())),
|
||||
"drift": numeric([x["drift"] for x in cells.values()]),
|
||||
"primary_pass_rate": numeric([x["pass_rate"] for x in all_primary]),
|
||||
"selected_count": numeric([x["selection"]["count"] for x in all_primary]),
|
||||
"layer1_steps": numeric([
|
||||
anchor["layer1"]["steps"] for cell in cells.values() for anchor in cell["anchors"]
|
||||
]),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--root", type=Path, required=True)
|
||||
p.add_argument("--ground-truth", type=Path, required=True)
|
||||
p.add_argument("--p3-root", type=Path, required=True)
|
||||
p.add_argument("--out", type=Path, required=True)
|
||||
args = p.parse_args()
|
||||
result = analyze(args.root, args.ground_truth, args.p3_root)
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n")
|
||||
print(json.dumps({"status": result["status"], "verdicts": result["verdicts"], "red_flags": result["sanity"]["red_flags"], "gpu": result["gpu"]}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
250
runs/opprof-phase6/opprof_phase6_client.py
Normal file
250
runs/opprof-phase6/opprof_phase6_client.py
Normal file
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exact C1 anchor replay using the pinned AITuner trace/worker/SLO paths."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
AITUNER_ROOT = Path(os.environ.get("AITUNER_ROOT", Path(__file__).resolve().parents[2]))
|
||||
sys.path.insert(0, str(AITUNER_ROOT / "src"))
|
||||
os.environ.setdefault("AITUNER_CODEX_BASE_URL", "http://127.0.0.1:1")
|
||||
|
||||
from aituner.slo import evaluate_request, summarize_evaluations # noqa: E402
|
||||
from aituner.spec import load_study_spec # noqa: E402
|
||||
from aituner.trace import load_trace_requests, select_requests_for_threshold # noqa: E402
|
||||
from aituner.worker import _probe_drain_deadline, _replay_requests # noqa: E402
|
||||
|
||||
|
||||
def atomic_json(path: Path, payload: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def numeric(values: list[float | int | None]) -> dict[str, Any]:
|
||||
finite = [float(x) for x in values if x is not None and math.isfinite(float(x))]
|
||||
return {
|
||||
"n": len(values), "finite_n": len(finite), "missing_n": len(values) - len(finite),
|
||||
"min": min(finite) if finite else None, "max": max(finite) if finite else None,
|
||||
"distinct_n": len(set(finite)),
|
||||
}
|
||||
|
||||
|
||||
def load_selected(study_path: Path, anchor: float):
|
||||
study = load_study_spec(study_path)
|
||||
window, requests = load_trace_requests(study, study_spec_path=study_path)
|
||||
selected = select_requests_for_threshold(requests, threshold=anchor)
|
||||
return study, window, requests, selected
|
||||
|
||||
|
||||
def selected_summary(selected, duration_s: float, tp: int) -> dict[str, Any]:
|
||||
ids = "\n".join(item.row_id for item in selected)
|
||||
arrival = "\n".join(f"{item.arrival_s:.12f}" for item in selected)
|
||||
lengths = "\n".join(str(item.prompt_tokens_hint) for item in selected)
|
||||
return {
|
||||
"count": len(selected),
|
||||
"offered_req_s": len(selected) / duration_s,
|
||||
"offered_req_s_per_gpu": len(selected) / duration_s / tp,
|
||||
"request_id_order_sha256": hashlib.sha256(ids.encode()).hexdigest(),
|
||||
"arrival_order_sha256": hashlib.sha256(arrival.encode()).hexdigest(),
|
||||
"raw_length_order_sha256": hashlib.sha256(lengths.encode()).hexdigest(),
|
||||
"arrival_s": numeric([item.arrival_s for item in selected]),
|
||||
"raw_input_tokens": numeric([item.prompt_tokens_hint for item in selected]),
|
||||
"long_gt4096": sum(int(item.prompt_tokens_hint or 0) > 4096 for item in selected),
|
||||
}
|
||||
|
||||
|
||||
def run_replay(args: argparse.Namespace, *, warmup: bool) -> dict[str, Any]:
|
||||
study_path = Path(args.study)
|
||||
study, window, _requests, selected = load_selected(study_path, args.anchor)
|
||||
if warmup:
|
||||
first = selected[:16]
|
||||
if not any(int(item.prompt_tokens_hint or 0) > 4096 for item in first):
|
||||
long_item = next(item for item in selected if int(item.prompt_tokens_hint or 0) > 4096)
|
||||
first = [*selected[:15], long_item]
|
||||
first = sorted({item.row_id: item for item in first}.values(), key=lambda item: item.arrival_s)
|
||||
if len(first) < 16:
|
||||
raise RuntimeError("warmup set has fewer than 16 unique requests")
|
||||
start = first[0].arrival_s
|
||||
selected = [dataclasses.replace(item, arrival_s=item.arrival_s - start) for item in first]
|
||||
|
||||
duration_s = float(window.window_end - window.window_start)
|
||||
interval_start_mono_ns = time.monotonic_ns()
|
||||
interval_start_wall_ns = time.time_ns()
|
||||
outcomes, early_stopped, early_stop_reason = _replay_requests(
|
||||
selected,
|
||||
base_url=args.base_url,
|
||||
timeout_s=study.engine.request_timeout_s,
|
||||
max_concurrency=study.trace.max_concurrency,
|
||||
target_pass_rate=(0.0 if warmup else study.slo.target_pass_rate),
|
||||
max_lag_s=study.trace.early_stop_max_lag_s,
|
||||
max_elapsed_s=(
|
||||
120.0 if warmup else _probe_drain_deadline(
|
||||
selected, study.slo, ceiling=study.trace.early_stop_max_elapsed_s
|
||||
)
|
||||
),
|
||||
evaluate_outcome=lambda outcome: evaluate_request(outcome, study.slo),
|
||||
drain_inflight_on_early_stop=True,
|
||||
)
|
||||
interval_end_mono_ns = time.monotonic_ns()
|
||||
interval_end_wall_ns = time.time_ns()
|
||||
evaluations, slo_summary = summarize_evaluations(outcomes, study.slo)
|
||||
by_id = {item.row_id: item for item in selected}
|
||||
details = []
|
||||
for outcome, evaluation in zip(outcomes, evaluations):
|
||||
request = by_id[outcome.request_id]
|
||||
details.append({
|
||||
"request_id": outcome.request_id,
|
||||
"sampling_u": request.sampling_u,
|
||||
"arrival_s": request.arrival_s,
|
||||
"raw_input_tokens": request.prompt_tokens_hint,
|
||||
"success": outcome.success,
|
||||
"ttft_ms": outcome.ttft_ms,
|
||||
"tpot_ms": outcome.tpot_ms,
|
||||
"completion_tokens": outcome.completion_tokens,
|
||||
"completion_tokens_source": outcome.completion_tokens_source,
|
||||
"slo_pass": evaluation.passed,
|
||||
"reasons": evaluation.reasons,
|
||||
"error": outcome.error,
|
||||
})
|
||||
out = Path(args.result_dir)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
with (out / "requests.jsonl").open("w") as f:
|
||||
for item in details:
|
||||
f.write(json.dumps(item, sort_keys=True) + "\n")
|
||||
summary = selected_summary(selected, duration_s, args.tp)
|
||||
exact = sum(
|
||||
item.success and item.completion_tokens_source == "usage" and item.completion_tokens == 128
|
||||
for item in outcomes
|
||||
)
|
||||
result = {
|
||||
"schema": 1,
|
||||
"kind": "warmup" if warmup else "anchor",
|
||||
"cell": args.cell,
|
||||
"anchor": args.anchor,
|
||||
"tp": args.tp,
|
||||
"mns": args.mns,
|
||||
"study_sha256": sha256_file(study_path),
|
||||
"interval": {
|
||||
"start_mono_ns": interval_start_mono_ns, "end_mono_ns": interval_end_mono_ns,
|
||||
"start_wall_ns": interval_start_wall_ns, "end_wall_ns": interval_end_wall_ns,
|
||||
"elapsed_s": (interval_end_mono_ns - interval_start_mono_ns) / 1e9,
|
||||
},
|
||||
"selection": summary,
|
||||
"observed_count": len(outcomes),
|
||||
"exact_output_count": exact,
|
||||
"slo_pass_count": slo_summary["slo_pass_count"],
|
||||
"pass_rate": slo_summary["slo_pass_rate"],
|
||||
"feasible": bool(slo_summary["feasible"]),
|
||||
"early_stopped": early_stopped,
|
||||
"early_stop_reason": early_stop_reason,
|
||||
"ttft_ms": numeric([item.ttft_ms for item in outcomes]),
|
||||
"tpot_ms": numeric([item.tpot_ms for item in outcomes]),
|
||||
"invariants": {
|
||||
"selected_nonempty": bool(selected),
|
||||
"outcomes_cover_selected": len(outcomes) == len(selected),
|
||||
"exact_output_or_failed": all(
|
||||
(not item.success) or (
|
||||
item.completion_tokens_source == "usage" and item.completion_tokens == 128
|
||||
) for item in outcomes
|
||||
),
|
||||
"raw_lengths_present": all(item.prompt_tokens_hint is not None for item in selected),
|
||||
"arrival_nondecreasing": all(
|
||||
b.arrival_s >= a.arrival_s for a, b in zip(selected, selected[1:])
|
||||
),
|
||||
"warmup_16": (len(outcomes) >= 16 if warmup else True),
|
||||
"warmup_exact_16": (exact >= 16 if warmup else True),
|
||||
"warmup_long": (
|
||||
any(int(item.prompt_tokens_hint or 0) > 4096 for item in selected)
|
||||
if warmup else True
|
||||
),
|
||||
},
|
||||
}
|
||||
atomic_json(out / "result.json", result)
|
||||
print(json.dumps({k: result[k] for k in ("cell", "anchor", "kind", "pass_rate", "feasible")}))
|
||||
if not all(result["invariants"].values()):
|
||||
raise RuntimeError(f"client invariants failed: {result['invariants']}")
|
||||
return result
|
||||
|
||||
|
||||
def preflight(args: argparse.Namespace) -> None:
|
||||
ground = json.loads(Path(args.ground_truth).read_text())
|
||||
studies = {1: Path(args.primary_study), 2: Path(args.primary_study), 4: Path(args.tp4_study)}
|
||||
loaded = {}
|
||||
mismatches = []
|
||||
values = []
|
||||
for cell in ground["cells"]:
|
||||
tp = int(cell["tensor_parallel_size"])
|
||||
if tp not in loaded:
|
||||
_study, _window, requests, _selected = load_selected(studies[tp], 0.0)
|
||||
loaded[tp] = requests
|
||||
for historical in cell["probe_history"]:
|
||||
selected = select_requests_for_threshold(
|
||||
loaded[tp], threshold=float(historical["sampling_u"])
|
||||
)
|
||||
values.append(len(selected))
|
||||
if len(selected) != int(historical["request_count"]):
|
||||
mismatches.append({
|
||||
"cell": cell["cell_id"], "anchor": historical["sampling_u"],
|
||||
"expected": historical["request_count"], "actual": len(selected),
|
||||
})
|
||||
result = {
|
||||
"schema": 1, "observations": len(values), "mismatches": mismatches,
|
||||
"request_counts": numeric(values),
|
||||
"invariants": {"observations_92": len(values) == 92, "counts_match": not mismatches},
|
||||
}
|
||||
atomic_json(Path(args.out), result)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
if not all(result["invariants"].values()):
|
||||
raise RuntimeError("preflight count reconstruction failed")
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser()
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
pf = sub.add_parser("preflight")
|
||||
pf.add_argument("--ground-truth", required=True)
|
||||
pf.add_argument("--primary-study", required=True)
|
||||
pf.add_argument("--tp4-study", required=True)
|
||||
pf.add_argument("--out", required=True)
|
||||
for name in ("warmup", "run-anchor"):
|
||||
q = sub.add_parser(name)
|
||||
q.add_argument("--study", required=True)
|
||||
q.add_argument("--cell", required=True)
|
||||
q.add_argument("--anchor", type=float, required=True)
|
||||
q.add_argument("--tp", type=int, required=True)
|
||||
q.add_argument("--mns", type=int, required=True)
|
||||
q.add_argument("--base-url", required=True)
|
||||
q.add_argument("--result-dir", required=True)
|
||||
return p
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parser().parse_args()
|
||||
if args.command == "preflight":
|
||||
preflight(args)
|
||||
else:
|
||||
run_replay(args, warmup=args.command == "warmup")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
492
runs/opprof-phase6/opprof_phase6_controller.py
Normal file
492
runs/opprof-phase6/opprof_phase6_controller.py
Normal file
@@ -0,0 +1,492 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Detached adaptive Phase-6 controller for the frozen 25-anchor surface."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
WORKDIR = Path("/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712")
|
||||
RUN_ROOT = WORKDIR / "runs/phase6"
|
||||
STATE = RUN_ROOT / "controller-state.json"
|
||||
SOURCE = Path("/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0")
|
||||
VENV = Path("/tmp/wjh-opprof-phase2-dash0-20260711/.venv")
|
||||
AITUNER = Path("/home/admin/cpfs/wjh/aituner/aituner")
|
||||
MODEL = Path("/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B")
|
||||
CLIENT = WORKDIR / "scripts/opprof_phase6_client.py"
|
||||
PRIMARY_STUDY = WORKDIR / "provenance/study-primary.json"
|
||||
TP4_STUDY = WORKDIR / "provenance/study-tp4.json"
|
||||
GROUND = WORKDIR / "provenance/ground_truth.json"
|
||||
GPU_LIMIT = 3.0
|
||||
CPU_MAP = {i: f"{20*i}-{20*i+19}" for i in range(8)}
|
||||
MARKER = "opprof-phase6-20260712"
|
||||
OWNED_PGIDS: set[int] = set()
|
||||
|
||||
|
||||
CELLS = {
|
||||
"tp1_mns8": {"tp": 1, "mns": 8, "lower": .21875, "peak": .2265625, "upper": .23046875},
|
||||
"tp1_mns16": {"tp": 1, "mns": 16, "lower": .2421875, "peak": .24609375, "upper": .25},
|
||||
"tp1_mns32": {"tp": 1, "mns": 32, "lower": .234375, "peak": .2421875, "upper": .24609375},
|
||||
"tp1_mns64": {"tp": 1, "mns": 64, "lower": .234375, "peak": .2421875, "upper": .24609375},
|
||||
"tp2_mns8": {"tp": 2, "mns": 8, "lower": .4921875, "peak": .49609375, "upper": .5},
|
||||
"tp2_mns16": {"tp": 2, "mns": 16, "lower": .4921875, "peak": .49609375, "upper": .5},
|
||||
"tp2_mns32": {"tp": 2, "mns": 32, "lower": .75, "peak": .75390625, "upper": .7578125},
|
||||
"tp2_mns64": {"tp": 2, "mns": 64, "lower": .5, "peak": .75, "upper": .75390625},
|
||||
"tp4_mns8": {"tp": 4, "mns": 8, "lower": .016055910008, "peak": .016591107009, "upper": .017126304009},
|
||||
"tp4_mns16": {"tp": 4, "mns": 16, "lower": .033182214016, "peak": .033717411016, "upper": .034252608017, "trap": True},
|
||||
"tp4_mns32": {"tp": 4, "mns": 32, "lower": .033182214016, "peak": .033717411016, "upper": .034252608017},
|
||||
"tp4_mns64": {"tp": 4, "mns": 64, "lower": .033182214016, "peak": .033717411016, "upper": .034252608017},
|
||||
}
|
||||
WAVES = [
|
||||
("W1-tp1", [("tp1_mns8", (0,)), ("tp1_mns16", (1,)), ("tp1_mns32", (2,)), ("tp1_mns64", (3,))], .35),
|
||||
("W2-tp2", [("tp2_mns8", (0,1)), ("tp2_mns16", (2,3)), ("tp2_mns32", (4,5)), ("tp2_mns64", (6,7))], .65),
|
||||
("W3-tp4-trap", [("tp4_mns8", (0,1,2,3)), ("tp4_mns16", (4,5,6,7))], .85),
|
||||
("W4-tp4", [("tp4_mns32", (0,1,2,3)), ("tp4_mns64", (4,5,6,7))], .65),
|
||||
]
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(value, sort_keys=True, indent=2) + "\n")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def load_state() -> dict[str, Any]:
|
||||
if STATE.exists():
|
||||
return json.loads(STATE.read_text())
|
||||
return {
|
||||
"schema": 1, "status": "initialized", "gpu_hours_total": 0.0,
|
||||
"completed_primary_anchors": 0, "completed_confirmations": 0,
|
||||
"waves": {}, "failures": [], "started_at": time.time(),
|
||||
}
|
||||
|
||||
|
||||
def save_state(state: dict[str, Any]) -> None:
|
||||
atomic_json(STATE, state)
|
||||
|
||||
|
||||
def cpu_mask(gpus: tuple[int, ...]) -> str:
|
||||
return ",".join(CPU_MAP[g] for g in gpus)
|
||||
|
||||
|
||||
def study_for(tp: int) -> Path:
|
||||
return TP4_STUDY if tp == 4 else PRIMARY_STUDY
|
||||
|
||||
|
||||
def run_text(command: list[str], check: bool = True) -> str:
|
||||
result = subprocess.run(command, text=True, capture_output=True)
|
||||
if check and result.returncode:
|
||||
raise RuntimeError(f"command failed {command}: {result.stderr}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def compute_pids() -> list[int]:
|
||||
text = run_text([
|
||||
"nvidia-smi", "--query-compute-apps=pid", "--format=csv,noheader,nounits"
|
||||
], check=False)
|
||||
return sorted({int(x.strip()) for x in text.splitlines() if x.strip().isdigit()})
|
||||
|
||||
|
||||
def pid_owned(pid: int) -> bool:
|
||||
try:
|
||||
if os.getpgid(pid) in OWNED_PGIDS:
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return True
|
||||
try:
|
||||
env = Path(f"/proc/{pid}/environ").read_bytes().split(b"\0")
|
||||
except (FileNotFoundError, PermissionError):
|
||||
return False
|
||||
return f"OPPROF_PHASE6_MARKER={MARKER}".encode() in env
|
||||
|
||||
|
||||
def assert_no_other_compute() -> None:
|
||||
other = [pid for pid in compute_pids() if not pid_owned(pid)]
|
||||
if other:
|
||||
raise RuntimeError(f"outside GPU processes detected: {other}")
|
||||
|
||||
|
||||
def assert_all_idle() -> None:
|
||||
if compute_pids():
|
||||
raise RuntimeError(f"GPU compute processes remain: {compute_pids()}")
|
||||
rows = run_text([
|
||||
"nvidia-smi", "--query-gpu=index,memory.used,utilization.gpu",
|
||||
"--format=csv,noheader,nounits",
|
||||
])
|
||||
bad = []
|
||||
for line in rows.splitlines():
|
||||
index, memory, util = [int(x.strip()) for x in line.split(",")]
|
||||
if memory != 0 or util != 0:
|
||||
bad.append((index, memory, util))
|
||||
if bad:
|
||||
raise RuntimeError(f"GPU cleanup failure: {bad}")
|
||||
|
||||
|
||||
def wait_ready(entry: dict[str, Any], timeout: float = 300.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
url = f"http://127.0.0.1:{entry['port']}/v1/models"
|
||||
while time.monotonic() < deadline:
|
||||
if entry["server"].poll() is not None:
|
||||
raise RuntimeError(f"server exited before ready: {entry['cell']}")
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status < 500:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
assert_no_other_compute()
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"server ready timeout: {entry['cell']}")
|
||||
|
||||
|
||||
def server_command(cell: str, gpus: tuple[int, ...], port: int) -> list[str]:
|
||||
cfg = CELLS[cell]
|
||||
return [
|
||||
"taskset", "-c", cpu_mask(gpus), str(VENV / "bin/vllm"), "serve", str(MODEL),
|
||||
"--host", "127.0.0.1", "--port", str(port),
|
||||
"--served-model-name", "qwen3-30b-a3b-community",
|
||||
"--max-num-batched-tokens", "8192", "--max-num-seqs", str(cfg["mns"]),
|
||||
"--tensor-parallel-size", str(cfg["tp"]), "--shutdown-timeout", "120",
|
||||
]
|
||||
|
||||
|
||||
def client_command(entry: dict[str, Any], anchor: float, out: Path, warmup: bool) -> list[str]:
|
||||
cfg = CELLS[entry["cell"]]
|
||||
return [
|
||||
"taskset", "-c", cpu_mask(entry["gpus"]), str(VENV / "bin/python"), str(CLIENT),
|
||||
"warmup" if warmup else "run-anchor", "--study", str(study_for(cfg["tp"])),
|
||||
"--cell", entry["cell"], "--anchor", str(anchor), "--tp", str(cfg["tp"]),
|
||||
"--mns", str(cfg["mns"]), "--base-url", f"http://127.0.0.1:{entry['port']}",
|
||||
"--result-dir", str(out),
|
||||
]
|
||||
|
||||
|
||||
def live_gpu_hours(entries: list[dict[str, Any]]) -> float:
|
||||
now = time.time()
|
||||
return sum(
|
||||
len(e["gpus"]) * ((e.get("stopped_at") or now) - e["spawned_at"])
|
||||
for e in entries
|
||||
) / 3600
|
||||
|
||||
|
||||
def run_clients(
|
||||
entries: list[dict[str, Any]], assignments: list[tuple[dict[str, Any], float, Path]],
|
||||
state: dict[str, Any], wave_name: str, warmup: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
processes = []
|
||||
handles = []
|
||||
for entry, anchor, out in assignments:
|
||||
command = client_command(entry, anchor, out, warmup)
|
||||
with (entry["dir"] / "commands.log").open("a") as f:
|
||||
f.write(f"CLIENT {shlex.join(command)}\n")
|
||||
handle = (out.parent / f"{out.name}.log").open("ab", buffering=0)
|
||||
handles.append(handle)
|
||||
client_env = os.environ.copy()
|
||||
client_env.update({"AITUNER_ROOT": str(AITUNER), "PYTHONUNBUFFERED": "1"})
|
||||
p = subprocess.Popen(
|
||||
command, cwd=WORKDIR, env=client_env, stdout=handle,
|
||||
stderr=subprocess.STDOUT, start_new_session=True,
|
||||
)
|
||||
processes.append((entry, anchor, out, p))
|
||||
deadline = time.monotonic() + 180
|
||||
while any(p.poll() is None for *_rest, p in processes):
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(f"client batch timeout: {wave_name}")
|
||||
for entry in entries:
|
||||
if entry.get("stopped_at") is None and entry["server"].poll() is not None:
|
||||
raise RuntimeError(f"server exited during client: {entry['cell']}")
|
||||
assert_no_other_compute()
|
||||
if state["gpu_hours_total"] + live_gpu_hours(entries) >= GPU_LIMIT:
|
||||
raise RuntimeError("3.0 H20-hour hard stop reached")
|
||||
time.sleep(1)
|
||||
for handle in handles:
|
||||
handle.close()
|
||||
bad = [(e["cell"], a, p.returncode) for e, a, _o, p in processes if p.returncode]
|
||||
if bad:
|
||||
raise RuntimeError(f"client failures: {bad}")
|
||||
results = []
|
||||
for entry, anchor, out, _p in processes:
|
||||
result = json.loads((out / "result.json").read_text())
|
||||
results.append(result)
|
||||
entry.setdefault("results", []).append({"anchor": anchor, "dir": str(out), "kind": result["kind"]})
|
||||
return results
|
||||
|
||||
|
||||
def stop_entry(entry: dict[str, Any]) -> None:
|
||||
if entry.get("stopped_at") is not None:
|
||||
return
|
||||
process = entry["server"]
|
||||
try:
|
||||
# Official vLLM shutdown: signal the API parent so EngineCore drains and
|
||||
# emits the in-stream footer/final sidecar. Process-group signals are
|
||||
# fallback only.
|
||||
os.kill(process.pid, signal.SIGINT)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=150)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
process.wait(timeout=30)
|
||||
entry["stopped_at"] = time.time()
|
||||
entry["server_handle"].close()
|
||||
|
||||
|
||||
def validate_cell(entry: dict[str, Any]) -> dict[str, Any]:
|
||||
log = (entry["dir"] / "server.log").read_text(errors="replace").splitlines()
|
||||
ready = [i for i, line in enumerate(log) if "Application startup complete" in line]
|
||||
event = re.compile(r"torch\.compile took|Directly load AOT compilation|\bCompiling\b|Capturing CUDA graphs", re.I)
|
||||
post_ready_events = [i + 1 for i, line in enumerate(log) if event.search(line) and ready and i > ready[0]]
|
||||
streams = sorted((entry["dir"] / "opprof").glob("*.jsonl"))
|
||||
sidecars = sorted((entry["dir"] / "opprof").glob("*.jsonl.footer.json"))
|
||||
if len(streams) != 1 or len(sidecars) != 1:
|
||||
raise RuntimeError(f"Layer1 stream/sidecar mismatch: {entry['cell']}")
|
||||
decoded = [json.loads(line) for line in streams[0].read_text().splitlines()]
|
||||
footers = [item for item in decoded if item.get("record_type") == "footer"]
|
||||
records = [item for item in decoded if "step_index" in item]
|
||||
sidecar = json.loads(sidecars[0].read_text())
|
||||
indices = [int(item["step_index"]) for item in records]
|
||||
warm = json.loads((entry["dir"] / "warmup/result.json").read_text())
|
||||
intervals_ok = True
|
||||
for item in entry.get("results", []):
|
||||
if item["kind"] != "anchor":
|
||||
continue
|
||||
result = json.loads((Path(item["dir"]) / "result.json").read_text())
|
||||
lo, hi = result["interval"]["start_mono_ns"], result["interval"]["end_mono_ns"]
|
||||
intervals_ok &= any(lo <= int(r["submit_mono_ns"]) <= hi for r in records)
|
||||
common = {
|
||||
"one_ready_marker": len(ready) == 1,
|
||||
"compile_capture_pre_ready": not post_ready_events,
|
||||
"warmup_exact_16": warm["exact_output_count"] >= 16,
|
||||
"warmup_long": warm["selection"]["long_gt4096"] >= 1,
|
||||
"layer1_contiguous": indices == list(range(len(indices))),
|
||||
"written_matches_records": sidecar.get("written_records") == len(records),
|
||||
"encoded_balanced": sidecar.get("encoded_records") == sidecar.get("written_records") + sidecar.get("dropped_records"),
|
||||
"last_step_matches": bool(records) and sidecar.get("last_step_index") == records[-1]["step_index"],
|
||||
"layer1_zero_drops": sidecar.get("dropped_records") == 0,
|
||||
"anchor_intervals_present": intervals_ok,
|
||||
}
|
||||
if footers:
|
||||
accounting = {
|
||||
"one_footer_last": len(footers) == 1 and decoded[-1] is footers[0],
|
||||
"sidecar_final": sidecar.get("final") is True,
|
||||
"footer_sidecar_agrees": all(
|
||||
footers[0].get(key) == sidecar.get(key)
|
||||
for key in ("encoded_records", "written_records", "dropped_records")
|
||||
),
|
||||
}
|
||||
accounting_mode = "graceful-footer"
|
||||
else:
|
||||
delta = abs(streams[0].stat().st_mtime_ns - int(sidecar["checkpoint_wall_ns"])) / 1e9
|
||||
accounting = {
|
||||
"checkpoint_sidecar": sidecar.get("final") is False,
|
||||
"checkpoint_within_flush_of_stream": delta <= float(sidecar["flush_interval_seconds"]) + .1,
|
||||
}
|
||||
accounting_mode = "checkpoint-sidecar-fallback"
|
||||
invariants = {**common, **accounting}
|
||||
if not all(invariants.values()):
|
||||
raise RuntimeError(f"cell validity failure {entry['cell']}: {invariants}")
|
||||
result = {
|
||||
"cell": entry["cell"], "invariants": invariants, "layer1_records": len(records),
|
||||
"stream": str(streams[0]), "post_ready_capture_events": post_ready_events,
|
||||
"accounting_mode": accounting_mode,
|
||||
}
|
||||
atomic_json(entry["dir"] / "cell-valid.json", result)
|
||||
return result
|
||||
|
||||
|
||||
def historical_expected() -> dict[tuple[str, float], dict[str, Any]]:
|
||||
ground = json.loads(GROUND.read_text())
|
||||
result = {}
|
||||
for cell in ground["cells"]:
|
||||
for probe in cell["probe_history"]:
|
||||
result[(cell["cell_id"], float(probe["sampling_u"]))] = probe
|
||||
return result
|
||||
|
||||
|
||||
def execute_wave(index: int, state: dict[str, Any], expected: dict[tuple[str, float], dict[str, Any]]) -> None:
|
||||
wave_name, assignments, estimate = WAVES[index]
|
||||
if state["waves"].get(wave_name, {}).get("status") == "complete":
|
||||
return
|
||||
future = sum(w[2] for w in WAVES[index:]) + .10
|
||||
if state["gpu_hours_total"] + future >= GPU_LIMIT:
|
||||
raise RuntimeError(f"projected budget exceeds cap before {wave_name}: {state['gpu_hours_total']+future}")
|
||||
echo = (
|
||||
f"WAVE_ECHO wave={wave_name} assignments="
|
||||
+ ",".join(f"{cell}:gpu{'+'.join(map(str, gpus))}" for cell, gpus in assignments)
|
||||
+ f" spent_h20h={state['gpu_hours_total']:.6f} wave_est_h20h={estimate:.3f} "
|
||||
+ f"remaining_projection_h20h={future:.3f} cap_h20h={GPU_LIMIT:.1f} "
|
||||
+ f"ground_truth={GROUND} workload=chat_w20260311_1000.jsonl"
|
||||
)
|
||||
with (RUN_ROOT / "launch-echo.log").open("a") as handle:
|
||||
handle.write(echo + "\n")
|
||||
print(echo, flush=True)
|
||||
assert_all_idle()
|
||||
wave_dir = RUN_ROOT / "waves" / wave_name
|
||||
wave_dir.mkdir(parents=True, exist_ok=True)
|
||||
entries = []
|
||||
state["status"] = "running"
|
||||
state["waves"][wave_name] = {"status": "starting", "estimate_h20_hours": estimate, "started_at": time.time()}
|
||||
save_state(state)
|
||||
failure = None
|
||||
try:
|
||||
for offset, (cell, gpus) in enumerate(assignments):
|
||||
cell_dir = RUN_ROOT / "cells" / cell
|
||||
cell_dir.mkdir(parents=True, exist_ok=True)
|
||||
port = 8500 + index * 10 + offset
|
||||
command = server_command(cell, gpus, port)
|
||||
with (cell_dir / "commands.log").open("a") as f:
|
||||
f.write(f"SERVER {shlex.join(command)}\n")
|
||||
handle = (cell_dir / "server.log").open("ab", buffering=0)
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"CUDA_VISIBLE_DEVICES": ",".join(map(str, gpus)),
|
||||
"VLLM_OPPROF_DIR": str(cell_dir / "opprof"),
|
||||
"OPPROF_PHASE6_MARKER": MARKER, "AITUNER_ROOT": str(AITUNER),
|
||||
"HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1",
|
||||
})
|
||||
server = subprocess.Popen(command, cwd=SOURCE, env=env, stdout=handle, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
OWNED_PGIDS.add(server.pid)
|
||||
entries.append({
|
||||
"cell": cell, "gpus": gpus, "port": port, "dir": cell_dir,
|
||||
"server": server, "server_handle": handle, "spawned_at": time.time(),
|
||||
})
|
||||
for entry in entries:
|
||||
wait_ready(entry)
|
||||
state["waves"][wave_name]["status"] = "warmup"
|
||||
save_state(state)
|
||||
run_clients(entries, [
|
||||
(e, CELLS[e["cell"]]["peak"], e["dir"] / "warmup") for e in entries
|
||||
], state, wave_name, warmup=True)
|
||||
state["waves"][wave_name]["status"] = "peaks"
|
||||
save_state(state)
|
||||
peaks = run_clients(entries, [
|
||||
(e, CELLS[e["cell"]]["peak"], e["dir"] / f"anchor-{CELLS[e['cell']]['peak']}")
|
||||
for e in entries
|
||||
], state, wave_name)
|
||||
for result in peaks:
|
||||
old = expected[(result["cell"], float(result["anchor"]))]
|
||||
if result["selection"]["count"] != old["request_count"]:
|
||||
raise RuntimeError(f"selection mismatch {result['cell']} peak")
|
||||
neighbor_jobs = []
|
||||
for entry, result in zip(entries, peaks, strict=True):
|
||||
key = "upper" if result["feasible"] else "lower"
|
||||
anchor = CELLS[entry["cell"]][key]
|
||||
neighbor_jobs.append((entry, anchor, entry["dir"] / f"anchor-{anchor}"))
|
||||
neighbors = run_clients(entries, neighbor_jobs, state, wave_name)
|
||||
for result in neighbors:
|
||||
old = expected[(result["cell"], float(result["anchor"]))]
|
||||
if result["selection"]["count"] != old["request_count"]:
|
||||
raise RuntimeError(f"selection mismatch {result['cell']} neighbor")
|
||||
trap_entry = next((e for e in entries if CELLS[e["cell"]].get("trap")), None)
|
||||
if trap_entry is not None:
|
||||
used = {float(item["anchor"]) for item in trap_entry["results"] if item["kind"] == "anchor"}
|
||||
extra = next(CELLS[trap_entry["cell"]][k] for k in ("lower", "upper") if CELLS[trap_entry["cell"]][k] not in used)
|
||||
extra_result = run_clients(entries, [(trap_entry, extra, trap_entry["dir"] / f"anchor-{extra}")], state, wave_name)[0]
|
||||
if extra_result["selection"]["count"] != expected[(extra_result["cell"], float(extra))]["request_count"]:
|
||||
raise RuntimeError("trap extra selection mismatch")
|
||||
|
||||
# Confirm only protocol-triggered anchors while the relevant server is hot.
|
||||
triggers = []
|
||||
for entry in entries:
|
||||
for item in entry.get("results", []):
|
||||
if item["kind"] != "anchor":
|
||||
continue
|
||||
result = json.loads((Path(item["dir"]) / "result.json").read_text())
|
||||
old = expected[(entry["cell"], float(result["anchor"]))]
|
||||
flip = bool(result["feasible"]) != bool(old["feasible"])
|
||||
if .93 <= float(result["pass_rate"]) <= .97 or (entry["cell"] in {"tp2_mns32", "tp4_mns16"} and flip):
|
||||
priority = 0 if entry["cell"] == "tp2_mns32" else (1 if entry["cell"] == "tp4_mns16" else 2)
|
||||
triggers.append((priority, entry, float(result["anchor"])))
|
||||
triggers.sort(key=lambda x: x[0])
|
||||
for _priority, entry, anchor in triggers:
|
||||
projected_extra = len(entry["gpus"]) * 80 / 3600
|
||||
future_primary = sum(w[2] for w in WAVES[index + 1:])
|
||||
if state["gpu_hours_total"] + live_gpu_hours(entries) + future_primary + projected_extra + .03 >= GPU_LIMIT:
|
||||
state.setdefault("unconfirmed_triggers", []).append({"cell": entry["cell"], "anchor": anchor})
|
||||
continue
|
||||
confirm_index = 1 + sum(1 for item in entry.get("results", []) if item["kind"] == "anchor" and Path(item["dir"]).name.startswith("confirm"))
|
||||
out = entry["dir"] / f"confirm-{confirm_index}-anchor-{anchor}"
|
||||
run_clients(entries, [(entry, anchor, out)], state, wave_name)
|
||||
state["completed_confirmations"] += 1
|
||||
state["waves"][wave_name]["status"] = "stopping"
|
||||
save_state(state)
|
||||
except Exception as error:
|
||||
failure = error
|
||||
finally:
|
||||
for entry in entries:
|
||||
try:
|
||||
stop_entry(entry)
|
||||
except Exception as error:
|
||||
failure = failure or error
|
||||
time.sleep(2)
|
||||
try:
|
||||
assert_all_idle()
|
||||
except Exception as error:
|
||||
failure = failure or error
|
||||
wave_hours = live_gpu_hours(entries)
|
||||
state["gpu_hours_total"] += wave_hours
|
||||
state["waves"][wave_name]["gpu_hours"] = wave_hours
|
||||
if failure is not None:
|
||||
state["waves"][wave_name]["status"] = "failed"
|
||||
state["waves"][wave_name]["failure"] = repr(failure)
|
||||
state["status"] = "failed"
|
||||
state["failures"].append({"wave": wave_name, "failure": repr(failure)})
|
||||
save_state(state)
|
||||
raise failure
|
||||
validations = [validate_cell(entry) for entry in entries]
|
||||
primary_count = sum(
|
||||
1 for entry in entries for item in entry.get("results", [])
|
||||
if item["kind"] == "anchor" and not Path(item["dir"]).name.startswith("confirm")
|
||||
)
|
||||
state["completed_primary_anchors"] += primary_count
|
||||
state["waves"][wave_name].update({
|
||||
"status": "complete", "completed_at": time.time(), "primary_anchors": primary_count,
|
||||
"validations": validations,
|
||||
})
|
||||
save_state(state)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--resume", action="store_true")
|
||||
args = parser.parse_args()
|
||||
RUN_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
state = load_state()
|
||||
expected = historical_expected()
|
||||
state["status"] = "running"
|
||||
save_state(state)
|
||||
for index in range(len(WAVES)):
|
||||
execute_wave(index, state, expected)
|
||||
state["status"] = "primary_complete"
|
||||
state["completed_at"] = time.time()
|
||||
save_state(state)
|
||||
print(json.dumps({
|
||||
"status": state["status"], "primary_anchors": state["completed_primary_anchors"],
|
||||
"confirmations": state["completed_confirmations"], "gpu_hours": state["gpu_hours_total"],
|
||||
}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
419
runs/opprof-phase6/opprof_phase6_solo_controller.py
Normal file
419
runs/opprof-phase6/opprof_phase6_solo_controller.py
Normal file
@@ -0,0 +1,419 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A-P6-2 serialized solo controller for authoritative Phase-6 frontiers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import opprof_phase6_controller as base
|
||||
|
||||
|
||||
SOLO_ROOT = base.RUN_ROOT / "solo-authoritative"
|
||||
STATE = SOLO_ROOT / "controller-state.json"
|
||||
CAMPAIGN_STATE = base.RUN_ROOT / "controller-state.json"
|
||||
GPU_LIMIT = 6.0
|
||||
SAFETY_HOURS = 0.20
|
||||
MARKER = "opprof-phase6-solo-A-P6-2"
|
||||
TRACE = base.AITUNER / "trace_windows/traces/chat_w20260311_1000.jsonl"
|
||||
|
||||
ORDER = [
|
||||
"tp4_mns32", "tp4_mns64", "tp2_mns32", "tp2_mns64",
|
||||
"tp4_mns16", "tp2_mns8", "tp2_mns16", "tp4_mns8",
|
||||
"tp1_mns8", "tp1_mns16", "tp1_mns32", "tp1_mns64",
|
||||
]
|
||||
|
||||
# Exact co-located primaries are remeasured before adaptive crawl. W4 had no
|
||||
# prior primary, so it starts at P and selects L/U from the solo result.
|
||||
CORE = {
|
||||
"tp1_mns8": [base.CELLS["tp1_mns8"]["peak"], base.CELLS["tp1_mns8"]["lower"]],
|
||||
"tp1_mns16": [base.CELLS["tp1_mns16"]["peak"], base.CELLS["tp1_mns16"]["upper"]],
|
||||
"tp1_mns32": [base.CELLS["tp1_mns32"]["peak"], base.CELLS["tp1_mns32"]["upper"]],
|
||||
"tp1_mns64": [base.CELLS["tp1_mns64"]["peak"], base.CELLS["tp1_mns64"]["upper"]],
|
||||
"tp2_mns8": [base.CELLS["tp2_mns8"]["peak"], base.CELLS["tp2_mns8"]["lower"]],
|
||||
"tp2_mns16": [base.CELLS["tp2_mns16"]["peak"], base.CELLS["tp2_mns16"]["lower"]],
|
||||
"tp2_mns32": [base.CELLS["tp2_mns32"]["peak"], base.CELLS["tp2_mns32"]["lower"]],
|
||||
"tp2_mns64": [base.CELLS["tp2_mns64"]["peak"], base.CELLS["tp2_mns64"]["lower"]],
|
||||
"tp4_mns8": [base.CELLS["tp4_mns8"]["peak"], base.CELLS["tp4_mns8"]["lower"]],
|
||||
"tp4_mns16": [
|
||||
base.CELLS["tp4_mns16"]["peak"], base.CELLS["tp4_mns16"]["lower"],
|
||||
base.CELLS["tp4_mns16"]["upper"],
|
||||
],
|
||||
"tp4_mns32": [base.CELLS["tp4_mns32"]["peak"]],
|
||||
"tp4_mns64": [base.CELLS["tp4_mns64"]["peak"]],
|
||||
}
|
||||
|
||||
CELL_ESTIMATE = {cell: {1: .11, 2: .22, 4: .48}[cfg["tp"]] for cell, cfg in base.CELLS.items()}
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: Any) -> None:
|
||||
base.atomic_json(path, value)
|
||||
|
||||
|
||||
def load_state() -> dict[str, Any]:
|
||||
if STATE.exists():
|
||||
return json.loads(STATE.read_text())
|
||||
campaign = json.loads(CAMPAIGN_STATE.read_text())
|
||||
return {
|
||||
"schema": 1, "amendment": "A-P6-2", "status": "initialized",
|
||||
"hard_cap_h20_hours": GPU_LIMIT,
|
||||
"prior_h20_hours": float(campaign["gpu_hours_total"]),
|
||||
"gpu_hours_total": float(campaign["gpu_hours_total"]),
|
||||
"solo_gpu_hours": 0.0, "completed_cells": 0,
|
||||
"primary_anchors": 0, "confirmations": 0,
|
||||
"cells": {}, "failures": [], "started_at": time.time(),
|
||||
}
|
||||
|
||||
|
||||
def save_state(state: dict[str, Any]) -> None:
|
||||
atomic_json(STATE, state)
|
||||
|
||||
|
||||
def historical() -> tuple[dict[tuple[str, float], dict[str, Any]], dict[str, list[float]]]:
|
||||
ground = json.loads(base.GROUND.read_text())
|
||||
expected = {}
|
||||
histories = {}
|
||||
for cell in ground["cells"]:
|
||||
anchors = []
|
||||
for probe in cell["probe_history"]:
|
||||
anchor = float(probe["sampling_u"])
|
||||
expected[(cell["cell_id"], anchor)] = probe
|
||||
anchors.append(anchor)
|
||||
histories[cell["cell_id"]] = sorted(anchors)
|
||||
return expected, histories
|
||||
|
||||
|
||||
def same_anchor(left: float, right: float) -> bool:
|
||||
return math.isclose(left, right, rel_tol=0, abs_tol=1e-15)
|
||||
|
||||
|
||||
def colocated_primary(cell: str, anchor: float) -> dict[str, Any] | None:
|
||||
cell_dir = base.RUN_ROOT / "cells" / cell
|
||||
for path in cell_dir.glob("anchor-*/result.json"):
|
||||
item = json.loads(path.read_text())
|
||||
if same_anchor(float(item["anchor"]), anchor):
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def append_echo(line: str) -> None:
|
||||
SOLO_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
with (SOLO_ROOT / "launch-echo.log").open("a") as handle:
|
||||
handle.write(line + "\n")
|
||||
print(line, flush=True)
|
||||
|
||||
|
||||
def wait_all_idle(timeout: float = 30.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
base.assert_all_idle()
|
||||
return
|
||||
except RuntimeError as current:
|
||||
error = current
|
||||
time.sleep(1)
|
||||
raise error or RuntimeError("GPU cleanup did not reach idle")
|
||||
|
||||
|
||||
def remaining_projection(index: int) -> float:
|
||||
return sum(CELL_ESTIMATE[cell] for cell in ORDER[index:]) + SAFETY_HOURS
|
||||
|
||||
|
||||
def start_entry(cell: str, index: int) -> dict[str, Any]:
|
||||
cfg = base.CELLS[cell]
|
||||
gpus = tuple(range(int(cfg["tp"])))
|
||||
cell_dir = SOLO_ROOT / "cells" / cell
|
||||
cell_dir.mkdir(parents=True, exist_ok=True)
|
||||
port = 8700 + index
|
||||
command = base.server_command(cell, gpus, port)
|
||||
with (cell_dir / "commands.log").open("a") as handle:
|
||||
handle.write(f"SERVER {shlex.join(command)}\n")
|
||||
server_handle = (cell_dir / "server.log").open("ab", buffering=0)
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"CUDA_VISIBLE_DEVICES": ",".join(map(str, gpus)),
|
||||
"VLLM_OPPROF_DIR": str(cell_dir / "opprof"),
|
||||
"OPPROF_PHASE6_MARKER": MARKER, "AITUNER_ROOT": str(base.AITUNER),
|
||||
"HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "PYTHONUNBUFFERED": "1",
|
||||
})
|
||||
server = subprocess.Popen(
|
||||
command, cwd=base.SOURCE, env=env, stdout=server_handle,
|
||||
stderr=subprocess.STDOUT, start_new_session=True,
|
||||
)
|
||||
base.OWNED_PGIDS.add(server.pid)
|
||||
return {
|
||||
"cell": cell, "gpus": gpus, "port": port, "dir": cell_dir,
|
||||
"server": server, "server_handle": server_handle,
|
||||
"spawned_at": time.time(), "results": [],
|
||||
}
|
||||
|
||||
|
||||
def run_one(
|
||||
entry: dict[str, Any], anchor: float, out: Path, state: dict[str, Any],
|
||||
cell_state: dict[str, Any], role: str,
|
||||
) -> dict[str, Any]:
|
||||
result = base.run_clients(
|
||||
[entry], [(entry, anchor, out)], state, f"solo-{entry['cell']}"
|
||||
)[0]
|
||||
expected_count = cell_state["expected_counts"][str(anchor)]
|
||||
if int(result["selection"]["count"]) != int(expected_count):
|
||||
raise RuntimeError(
|
||||
f"selection mismatch {entry['cell']} {anchor}: "
|
||||
f"{result['selection']['count']} != {expected_count}"
|
||||
)
|
||||
cell_state.setdefault("runs", []).append({
|
||||
"anchor": anchor, "role": role, "dir": str(out),
|
||||
"pass_rate": result["pass_rate"], "feasible": result["feasible"],
|
||||
})
|
||||
save_state(state)
|
||||
return result
|
||||
|
||||
|
||||
def anchor_trials(cell_state: dict[str, Any], anchor: float) -> list[dict[str, Any]]:
|
||||
return [
|
||||
item for item in cell_state.get("runs", [])
|
||||
if same_anchor(float(item["anchor"]), anchor)
|
||||
]
|
||||
|
||||
|
||||
def accepted_feasible(cell_state: dict[str, Any], anchor: float) -> bool | None:
|
||||
trials = anchor_trials(cell_state, anchor)
|
||||
votes = [bool(item["feasible"]) for item in trials]
|
||||
if not votes:
|
||||
return None
|
||||
if len(votes) == 1 or len(set(votes)) == 1:
|
||||
return votes[0]
|
||||
if len(votes) >= 3:
|
||||
return sum(votes) >= 2
|
||||
return None
|
||||
|
||||
|
||||
def optional_fits(
|
||||
state: dict[str, Any], entry: dict[str, Any], future_after: float,
|
||||
) -> bool:
|
||||
replay = len(entry["gpus"]) * 80 / 3600
|
||||
projected = (
|
||||
float(state["gpu_hours_total"]) + base.live_gpu_hours([entry])
|
||||
+ future_after + replay + SAFETY_HOURS
|
||||
)
|
||||
return projected < GPU_LIMIT
|
||||
|
||||
|
||||
def maybe_confirm(
|
||||
entry: dict[str, Any], anchor: float, primary: dict[str, Any],
|
||||
state: dict[str, Any], cell_state: dict[str, Any], expected: dict[tuple[str, float], dict[str, Any]],
|
||||
future_after: float,
|
||||
) -> None:
|
||||
old = expected[(entry["cell"], anchor)]
|
||||
coloc = colocated_primary(entry["cell"], anchor)
|
||||
disagreement = (
|
||||
bool(primary["feasible"]) != bool(old["feasible"])
|
||||
or (coloc is not None and bool(primary["feasible"]) != bool(coloc["feasible"]))
|
||||
)
|
||||
boundary = .93 <= float(primary["pass_rate"]) <= .97
|
||||
if not (disagreement or boundary):
|
||||
return
|
||||
while len(anchor_trials(cell_state, anchor)) < 3:
|
||||
trials = anchor_trials(cell_state, anchor)
|
||||
if len(trials) >= 2 and len({bool(item["feasible"]) for item in trials}) == 1:
|
||||
return
|
||||
if not optional_fits(state, entry, future_after):
|
||||
cell_state.setdefault("deferred_confirmations", []).append(anchor)
|
||||
return
|
||||
trial = len(trials) + 1
|
||||
out = entry["dir"] / f"confirm-{trial - 1}-anchor-{anchor}"
|
||||
run_one(entry, anchor, out, state, cell_state, f"confirmation-{trial}")
|
||||
state["confirmations"] += 1
|
||||
|
||||
|
||||
def run_primary(
|
||||
entry: dict[str, Any], anchor: float, state: dict[str, Any],
|
||||
cell_state: dict[str, Any], expected: dict[tuple[str, float], dict[str, Any]],
|
||||
future_after: float, role: str,
|
||||
) -> dict[str, Any]:
|
||||
existing = [item for item in cell_state.get("runs", []) if item["role"].startswith("primary")]
|
||||
if any(same_anchor(float(item["anchor"]), anchor) for item in existing):
|
||||
path = next(
|
||||
Path(item["dir"]) for item in existing
|
||||
if same_anchor(float(item["anchor"]), anchor)
|
||||
)
|
||||
return json.loads((path / "result.json").read_text())
|
||||
out = entry["dir"] / f"anchor-{anchor}"
|
||||
result = run_one(entry, anchor, out, state, cell_state, role)
|
||||
state["primary_anchors"] += 1
|
||||
maybe_confirm(entry, anchor, result, state, cell_state, expected, future_after)
|
||||
return result
|
||||
|
||||
|
||||
def next_below(history: list[float], tested: set[float]) -> float | None:
|
||||
if not tested:
|
||||
return None
|
||||
candidates = [x for x in history if x < min(tested) and x not in tested]
|
||||
return max(candidates) if candidates else None
|
||||
|
||||
|
||||
def next_above(history: list[float], tested: set[float]) -> float | None:
|
||||
if not tested:
|
||||
return None
|
||||
candidates = [x for x in history if x > max(tested) and x not in tested]
|
||||
return min(candidates) if candidates else None
|
||||
|
||||
|
||||
def execute_cell(
|
||||
index: int, cell: str, state: dict[str, Any],
|
||||
expected: dict[tuple[str, float], dict[str, Any]], histories: dict[str, list[float]],
|
||||
) -> None:
|
||||
if state["cells"].get(cell, {}).get("status") == "complete":
|
||||
return
|
||||
future = remaining_projection(index)
|
||||
if float(state["gpu_hours_total"]) + future >= GPU_LIMIT:
|
||||
state["status"] = "budget_projection_stop"
|
||||
state["budget_stop"] = {
|
||||
"before_cell": cell, "spent_h20_hours": state["gpu_hours_total"],
|
||||
"remaining_projection_h20_hours": future,
|
||||
"projected_total_h20_hours": state["gpu_hours_total"] + future,
|
||||
"hard_cap_h20_hours": GPU_LIMIT,
|
||||
}
|
||||
save_state(state)
|
||||
raise RuntimeError(f"projected budget exceeds cap before {cell}")
|
||||
cfg = base.CELLS[cell]
|
||||
echo = (
|
||||
f"SOLO_WAVE_ECHO cell={cell} tp={cfg['tp']} mns={cfg['mns']} "
|
||||
f"gpus=0-{int(cfg['tp'])-1} mandatory={','.join(map(str, CORE[cell]))} "
|
||||
f"spent_h20h={state['gpu_hours_total']:.6f} cell_est_h20h={CELL_ESTIMATE[cell]:.3f} "
|
||||
f"remaining_projection_h20h={future:.3f} cap_h20h={GPU_LIMIT:.1f} "
|
||||
f"ground_truth={base.GROUND} trace={TRACE}"
|
||||
)
|
||||
append_echo(echo)
|
||||
wait_all_idle()
|
||||
cell_state = {
|
||||
"status": "starting", "started_at": time.time(), "tp": cfg["tp"], "mns": cfg["mns"],
|
||||
"mandatory": CORE[cell],
|
||||
"expected_counts": {
|
||||
str(anchor): expected[(cell, anchor)]["request_count"] for anchor in histories[cell]
|
||||
},
|
||||
"runs": [],
|
||||
}
|
||||
state["status"] = "running"
|
||||
state["cells"][cell] = cell_state
|
||||
save_state(state)
|
||||
entry = start_entry(cell, index)
|
||||
failure = None
|
||||
future_after = sum(CELL_ESTIMATE[item] for item in ORDER[index + 1:])
|
||||
try:
|
||||
base.wait_ready(entry)
|
||||
cell_state["status"] = "warmup"
|
||||
save_state(state)
|
||||
warm = base.run_clients(
|
||||
[entry], [(entry, cfg["peak"], entry["dir"] / "warmup")],
|
||||
state, f"solo-{cell}", warmup=True,
|
||||
)[0]
|
||||
cell_state["warmup"] = {
|
||||
"exact_output_count": warm["exact_output_count"],
|
||||
"long_gt4096": warm["selection"]["long_gt4096"],
|
||||
}
|
||||
cell_state["status"] = "mandatory"
|
||||
save_state(state)
|
||||
for anchor in CORE[cell]:
|
||||
run_primary(entry, anchor, state, cell_state, expected, future_after, "primary-mandatory")
|
||||
|
||||
peak = float(cfg["peak"])
|
||||
peak_vote = accepted_feasible(cell_state, peak)
|
||||
if cell in {"tp4_mns32", "tp4_mns64"} and peak_vote is not None:
|
||||
direction = float(cfg["upper"] if peak_vote else cfg["lower"])
|
||||
run_primary(entry, direction, state, cell_state, expected, future_after, "primary-direction")
|
||||
|
||||
cell_state["status"] = "crawl"
|
||||
save_state(state)
|
||||
while True:
|
||||
primary_anchors = {
|
||||
float(item["anchor"]) for item in cell_state["runs"]
|
||||
if item["role"].startswith("primary")
|
||||
}
|
||||
votes = {anchor: accepted_feasible(cell_state, anchor) for anchor in primary_anchors}
|
||||
pass_anchors = [anchor for anchor, vote in votes.items() if vote is True]
|
||||
fail_anchors = [anchor for anchor, vote in votes.items() if vote is False]
|
||||
if pass_anchors and fail_anchors and max(pass_anchors) < min(fail_anchors):
|
||||
break
|
||||
if any(vote is None for vote in votes.values()):
|
||||
cell_state["censor"] = "UNRESOLVED_SOLO_ANCHOR"
|
||||
break
|
||||
if pass_anchors and not fail_anchors:
|
||||
candidate = next_above(histories[cell], primary_anchors)
|
||||
elif fail_anchors and not pass_anchors:
|
||||
candidate = next_below(histories[cell], primary_anchors)
|
||||
else:
|
||||
# Non-monotonic anchors already contain both states but no valid bracket.
|
||||
cell_state["censor"] = "NONMONOTONIC_SOLO_ANCHORS"
|
||||
break
|
||||
if candidate is None:
|
||||
cell_state["censor"] = "HISTORY_EDGE"
|
||||
break
|
||||
if not optional_fits(state, entry, future_after):
|
||||
cell_state["censor"] = "BUDGET_CENSORED"
|
||||
break
|
||||
run_primary(entry, candidate, state, cell_state, expected, future_after, "primary-crawl")
|
||||
cell_state["status"] = "stopping"
|
||||
save_state(state)
|
||||
except Exception as error:
|
||||
failure = error
|
||||
finally:
|
||||
try:
|
||||
base.stop_entry(entry)
|
||||
except Exception as error:
|
||||
failure = failure or error
|
||||
time.sleep(2)
|
||||
try:
|
||||
wait_all_idle()
|
||||
except Exception as error:
|
||||
failure = failure or error
|
||||
hours = base.live_gpu_hours([entry])
|
||||
state["gpu_hours_total"] += hours
|
||||
state["solo_gpu_hours"] += hours
|
||||
cell_state["gpu_hours"] = hours
|
||||
if failure is not None:
|
||||
cell_state["status"] = "failed"
|
||||
cell_state["failure"] = repr(failure)
|
||||
state["status"] = "failed"
|
||||
state["failures"].append({"cell": cell, "failure": repr(failure)})
|
||||
save_state(state)
|
||||
raise failure
|
||||
validation = base.validate_cell(entry)
|
||||
cell_state["validation"] = validation
|
||||
cell_state["status"] = "complete"
|
||||
cell_state["completed_at"] = time.time()
|
||||
state["completed_cells"] += 1
|
||||
save_state(state)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
SOLO_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
base.GPU_LIMIT = GPU_LIMIT
|
||||
base.MARKER = MARKER
|
||||
expected, histories = historical()
|
||||
state = load_state()
|
||||
state["status"] = "running"
|
||||
save_state(state)
|
||||
for index, cell in enumerate(ORDER):
|
||||
execute_cell(index, cell, state, expected, histories)
|
||||
state["status"] = "complete"
|
||||
state["completed_at"] = time.time()
|
||||
save_state(state)
|
||||
print(json.dumps({
|
||||
"status": state["status"], "cells": state["completed_cells"],
|
||||
"primary_anchors": state["primary_anchors"],
|
||||
"confirmations": state["confirmations"],
|
||||
"solo_gpu_hours": state["solo_gpu_hours"],
|
||||
"campaign_gpu_hours": state["gpu_hours_total"],
|
||||
}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"admission_stop_s": 363.15318555399426,
|
||||
"arrival": "steady",
|
||||
"clean": {
|
||||
"admitted": 113,
|
||||
"completed": 111,
|
||||
"completed_throughput_rps": 0.4625,
|
||||
"duration_s": 240.0,
|
||||
"end_s": 300.0,
|
||||
"failed": 0,
|
||||
"input_tokens": 968714,
|
||||
"offered_rps": 0.4708333333333333,
|
||||
"output_tokens": 25409,
|
||||
"start_s": 60.0
|
||||
},
|
||||
"clean_segment_seconds": 80.0,
|
||||
"drain_seconds": 1.3658369119802956,
|
||||
"elapsed_seconds": 364.51902246597456,
|
||||
"failed_records": 0,
|
||||
"load_point": "moderate",
|
||||
"manifest_admitted": 172,
|
||||
"manifest_exhausted": false,
|
||||
"manifest_rows": 4011,
|
||||
"manifest_sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33",
|
||||
"manifest_wrapped": false,
|
||||
"max_in_flight": 7,
|
||||
"num_clean_segments": 3,
|
||||
"profiles": [
|
||||
{
|
||||
"start_call_s": 300.00394913199125,
|
||||
"start_return_s": 300.03973603399936,
|
||||
"start_status": 200,
|
||||
"stop_call_s": 300.89519165997626,
|
||||
"stop_return_s": 301.6286224719952,
|
||||
"stop_status": 200,
|
||||
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783849466906401944.pt.trace.json.gz",
|
||||
"trace_ready_s": 300.8951868820004,
|
||||
"trace_sha256": "a15670f86d843d40e7c4b6f354dd0193ea1259824a96440392d923f5270ba095",
|
||||
"window": 1
|
||||
},
|
||||
{
|
||||
"start_call_s": 331.62960390897933,
|
||||
"start_return_s": 331.63382844498847,
|
||||
"start_status": 200,
|
||||
"stop_call_s": 332.4971265429922,
|
||||
"stop_return_s": 333.15040952397976,
|
||||
"stop_status": 200,
|
||||
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783849498465727706.pt.trace.json.gz",
|
||||
"trace_ready_s": 332.4971234249824,
|
||||
"trace_sha256": "0f5962e193e09a9b2f1df6cea23f92419dcf2b322ee4ef2135a4fc3e1fe0617c",
|
||||
"window": 2
|
||||
}
|
||||
],
|
||||
"rate_fraction": 0.6,
|
||||
"records": 172,
|
||||
"request_rate": 0.4725,
|
||||
"schema": 1,
|
||||
"segments": [
|
||||
{
|
||||
"admitted": 38,
|
||||
"completed": 37,
|
||||
"completed_throughput_rps": 0.4625,
|
||||
"duration_s": 80.0,
|
||||
"end_s": 140.0,
|
||||
"failed": 0,
|
||||
"input_tokens": 404192,
|
||||
"name": "A",
|
||||
"offered_rps": 0.475,
|
||||
"output_tokens": 8525,
|
||||
"start_s": 60.0
|
||||
},
|
||||
{
|
||||
"admitted": 37,
|
||||
"completed": 38,
|
||||
"completed_throughput_rps": 0.475,
|
||||
"duration_s": 80.0,
|
||||
"end_s": 220.0,
|
||||
"failed": 0,
|
||||
"input_tokens": 327301,
|
||||
"name": "B",
|
||||
"offered_rps": 0.4625,
|
||||
"output_tokens": 8617,
|
||||
"start_s": 140.0
|
||||
},
|
||||
{
|
||||
"admitted": 38,
|
||||
"completed": 36,
|
||||
"completed_throughput_rps": 0.45,
|
||||
"duration_s": 80.0,
|
||||
"end_s": 300.0,
|
||||
"failed": 0,
|
||||
"input_tokens": 237221,
|
||||
"name": "C",
|
||||
"offered_rps": 0.475,
|
||||
"output_tokens": 8267,
|
||||
"start_s": 220.0
|
||||
}
|
||||
],
|
||||
"successful_records": 172,
|
||||
"t0_mono_ns": 180739547167834,
|
||||
"t0_wall_ns": 1783849166673595809,
|
||||
"warmup_seconds": 60.0
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.328174978,
|
||||
"end_mono_ns": 199601637648726,
|
||||
"end_wall_ns": 1783868028764076133,
|
||||
"start_mono_ns": 199540309473748,
|
||||
"start_wall_ns": 1783867967435901551
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 39.29021051171873,
|
||||
"min": 4.445230456776771,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1787.3226249939762,
|
||||
"min": 37.47074800776318,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.471211524,
|
||||
"end_mono_ns": 199668842271892,
|
||||
"end_wall_ns": 1783868095968699176,
|
||||
"start_mono_ns": 199607371060368,
|
||||
"start_wall_ns": 1783868034497487431
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 143,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 44.43295162989699,
|
||||
"min": 4.492281684945301,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 1784.063341008732,
|
||||
"min": 57.29520702152513,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
21
runs/opprof-phase6/phase6/cells/tp1_mns16/cell-valid.json
Normal file
21
runs/opprof-phase6/phase6/cells/tp1_mns16/cell-valid.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"accounting_mode": "A-P6-1-checkpoint-sidecar",
|
||||
"cell": "tp1_mns16",
|
||||
"invariants": {
|
||||
"all_anchor_intervals_covered": true,
|
||||
"all_schema_1": true,
|
||||
"checkpoint_after_all_anchor_intervals": true,
|
||||
"checkpoint_sidecar": true,
|
||||
"checkpoint_within_flush_of_stream": true,
|
||||
"complete_final_newline": true,
|
||||
"encoded_balanced": true,
|
||||
"last_step_matches": true,
|
||||
"no_in_stream_footer": true,
|
||||
"steps_contiguous": true,
|
||||
"two_anchor_intervals": true,
|
||||
"written_matches_records": true,
|
||||
"zero_drops": true
|
||||
},
|
||||
"layer1_records": 9212,
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2638886-1783867898065648658.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp1_mns16/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp1_mns16/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8501 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/warmup
|
||||
CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/anchor-0.24609375
|
||||
CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.25 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/anchor-0.25
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638886-1783867898065648658.jsonl","encoded_records":9212,"written_records":9212,"dropped_records":0,"last_step_index":9211,"checkpoint_wall_ns":1783868096969027153,"flush_interval_seconds":1.0,"final":false}
|
||||
442
runs/opprof-phase6/phase6/cells/tp1_mns16/server.log
Normal file
442
runs/opprof-phase6/phase6/cells/tp1_mns16/server.log
Normal file
@@ -0,0 +1,442 @@
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8501, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 16, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 32, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:05 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:37021 backend=nccl
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:05 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:01<00:15, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:02<00:14, 1.07s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:03<00:13, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:04<00:13, 1.10s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:05<00:11, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:06<00:10, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:07<00:09, 1.10s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:08<00:08, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:09<00:07, 1.08s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:10<00:06, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:11<00:05, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:12<00:04, 1.04s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:13<00:03, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:15<00:02, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:16<00:01, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.17it/s]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.03s/it]
|
||||
(EngineCore pid=2638886)
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.52 seconds
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.268242 seconds
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/f4a50989f8/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.31 s
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.252 s
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:31 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/fe5a82dbe929018b7aea7dee05b5cd31a21fe2682aca78ee4cbf2b37c8a086d6/rank_0_0/model
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:31 [monitor.py:53] torch.compile took 5.96 s in total
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:32 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.20 s
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:32 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.43 GiB
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 321,456 tokens
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.85x
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2638886) 2026-07-12 14:51:35,155 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2638886) 2026-07-12 14:51:35,199 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2638886)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 14%|█▍ | 1/7 [00:00<00:00, 9.98it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 43%|████▎ | 3/7 [00:00<00:00, 10.33it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 71%|███████▏ | 5/7 [00:00<00:00, 9.18it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▌ | 6/7 [00:00<00:00, 8.44it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00, 7.90it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00, 8.49it/s]
|
||||
(EngineCore pid=2638886)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 40%|████ | 2/5 [00:00<00:00, 10.74it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 80%|████████ | 4/5 [00:00<00:00, 11.19it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 11.25it/s]
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:37 [gpu_model_runner.py:6660] Graph capturing finished in 2 secs, took 0.10 GiB
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:37 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.02 GiB (15.7%).
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:37 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:37 [core.py:337] init engine (profile, create kv cache, warmup model) took 11.83 s (compilation: 5.96 s)
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:38 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2638886-1783867898065648658.jsonl
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:38 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:38 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2638187) WARNING 07-12 14:51:38 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8501
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2638187) INFO: Started server process [2638187]
|
||||
(APIServer pid=2638187) INFO: Waiting for application startup.
|
||||
(APIServer pid=2638187) INFO: Application startup complete.
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52764 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638886) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638886) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:52:39 [loggers.py:273] Engine 000: Avg prompt throughput: 5299.4 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:52:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8.5 tokens/s, Avg generation throughput: 67.6 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 35.1%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:52:59 [loggers.py:273] Engine 000: Avg prompt throughput: 2139.7 tokens/s, Avg generation throughput: 267.4 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 43.4%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5435.1 tokens/s, Avg generation throughput: 320.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 33.3%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48094 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:19 [loggers.py:273] Engine 000: Avg prompt throughput: 4881.2 tokens/s, Avg generation throughput: 164.9 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.6%, Prefix cache hit rate: 29.1%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39864 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:29 [loggers.py:273] Engine 000: Avg prompt throughput: 6255.1 tokens/s, Avg generation throughput: 265.4 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.7%, Prefix cache hit rate: 24.2%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:39 [loggers.py:273] Engine 000: Avg prompt throughput: 9198.7 tokens/s, Avg generation throughput: 388.3 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.2%, Prefix cache hit rate: 20.8%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:43012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:43016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:43018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:43030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:49 [loggers.py:273] Engine 000: Avg prompt throughput: 6382.3 tokens/s, Avg generation throughput: 331.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.1%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53630 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:59 [loggers.py:273] Engine 000: Avg prompt throughput: 3970.7 tokens/s, Avg generation throughput: 126.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.3%, Prefix cache hit rate: 17.8%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54088 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:09 [loggers.py:273] Engine 000: Avg prompt throughput: 6063.3 tokens/s, Avg generation throughput: 290.7 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.9%, Prefix cache hit rate: 17.1%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54104 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:19 [loggers.py:273] Engine 000: Avg prompt throughput: 4203.4 tokens/s, Avg generation throughput: 273.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 16.7%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56492 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:29 [loggers.py:273] Engine 000: Avg prompt throughput: 4720.8 tokens/s, Avg generation throughput: 256.7 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 16.3%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:39 [loggers.py:273] Engine 000: Avg prompt throughput: 8295.5 tokens/s, Avg generation throughput: 280.7 tokens/s, Running: 16 reqs, Waiting: 1 reqs, GPU KV cache usage: 15.5%, Prefix cache hit rate: 15.5%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8540.1 tokens/s, Avg generation throughput: 333.9 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 17.1%, Prefix cache hit rate: 14.6%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp1_mns16/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp1_mns16/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp1_mns16/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp1_mns16/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.031388113,
|
||||
"end_mono_ns": 199530259142500,
|
||||
"end_wall_ns": 1783867957385569525,
|
||||
"start_mono_ns": 199522227754387,
|
||||
"start_wall_ns": 1783867949354181463
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 16,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 28.370322086701652,
|
||||
"min": 4.984751267873821,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 696.6323160158936,
|
||||
"min": 74.85444800113328,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.2421875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 137,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.294849452,
|
||||
"end_mono_ns": 199601604247251,
|
||||
"end_wall_ns": 1783868028730674502,
|
||||
"start_mono_ns": 199540309397799,
|
||||
"start_wall_ns": 1783867967435824912
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 137,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "e8e91fd47d9152811ed7bd79e20c0f45ba6677560a419542d9c17abd82bebb4b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"count": 137,
|
||||
"long_gt4096": 47,
|
||||
"offered_req_s": 2.283333333333333,
|
||||
"offered_req_s_per_gpu": 2.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 137,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"raw_length_order_sha256": "93176915562ff9a118f3550157ddd1025d73a6b70cf4d03be9765de9a1b5d744",
|
||||
"request_id_order_sha256": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017"
|
||||
},
|
||||
"slo_pass_count": 137,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 35.17025839386134,
|
||||
"min": 4.435019960625127,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 1485.6346309825312,
|
||||
"min": 34.36101900297217,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.288241408,
|
||||
"end_mono_ns": 199668653603418,
|
||||
"end_wall_ns": 1783868095780030817,
|
||||
"start_mono_ns": 199607365362010,
|
||||
"start_wall_ns": 1783868034491789246
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 45.32469625197036,
|
||||
"min": 4.495332385807511,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1799.771893012803,
|
||||
"min": 49.012041999958456,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
21
runs/opprof-phase6/phase6/cells/tp1_mns32/cell-valid.json
Normal file
21
runs/opprof-phase6/phase6/cells/tp1_mns32/cell-valid.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"accounting_mode": "A-P6-1-checkpoint-sidecar",
|
||||
"cell": "tp1_mns32",
|
||||
"invariants": {
|
||||
"all_anchor_intervals_covered": true,
|
||||
"all_schema_1": true,
|
||||
"checkpoint_after_all_anchor_intervals": true,
|
||||
"checkpoint_sidecar": true,
|
||||
"checkpoint_within_flush_of_stream": true,
|
||||
"complete_final_newline": true,
|
||||
"encoded_balanced": true,
|
||||
"last_step_matches": true,
|
||||
"no_in_stream_footer": true,
|
||||
"steps_contiguous": true,
|
||||
"two_anchor_intervals": true,
|
||||
"written_matches_records": true,
|
||||
"zero_drops": true
|
||||
},
|
||||
"layer1_records": 9633,
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2638900-1783867898685556703.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp1_mns32/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp1_mns32/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8502 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 32 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.2421875 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/warmup
|
||||
CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.2421875 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/anchor-0.2421875
|
||||
CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.24609375 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/anchor-0.24609375
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638900-1783867898685556703.jsonl","encoded_records":9633,"written_records":9633,"dropped_records":0,"last_step_index":9632,"checkpoint_wall_ns":1783868096780742186,"flush_interval_seconds":1.0,"final":false}
|
||||
436
runs/opprof-phase6/phase6/cells/tp1_mns32/server.log
Normal file
436
runs/opprof-phase6/phase6/cells/tp1_mns32/server.log
Normal file
@@ -0,0 +1,436 @@
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8502, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 32, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:05 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:50247 backend=nccl
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:05 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:01<00:16, 1.07s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:02<00:14, 1.07s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:03<00:13, 1.06s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:04<00:13, 1.10s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:05<00:11, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:06<00:10, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:07<00:09, 1.10s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:08<00:08, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:09<00:07, 1.08s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:10<00:06, 1.06s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:11<00:05, 1.06s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:12<00:04, 1.04s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:13<00:03, 1.06s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:15<00:02, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:16<00:01, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.18it/s]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.03s/it]
|
||||
(EngineCore pid=2638900)
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.49 seconds
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.231743 seconds
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/30ef41b5f5/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.33 s
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.266 s
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:32 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/738c624149a63d13eaf115eec4d2189ece948ac500e524d3e06e801d9915352d/rank_0_0/model
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:32 [monitor.py:53] torch.compile took 6.02 s in total
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:32 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.20 s
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:33 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=11 (largest=64), FULL=7 (largest=32)
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.11 GiB total
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.4 GiB
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9188 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9212. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 321,136 tokens
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.84x
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2638900) 2026-07-12 14:51:35,118 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2638900) 2026-07-12 14:51:35,162 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2638900)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 9%|▉ | 1/11 [00:00<00:01, 9.36it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 18%|█▊ | 2/11 [00:00<00:00, 9.41it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 3/11 [00:00<00:00, 9.30it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 36%|███▋ | 4/11 [00:00<00:00, 9.23it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 5/11 [00:00<00:00, 9.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 55%|█████▍ | 6/11 [00:00<00:00, 9.24it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 64%|██████▎ | 7/11 [00:00<00:00, 8.56it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 73%|███████▎ | 8/11 [00:00<00:00, 8.19it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 9/11 [00:01<00:00, 7.88it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 91%|█████████ | 10/11 [00:01<00:00, 7.84it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:01<00:00, 7.60it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:01<00:00, 8.31it/s]
|
||||
(EngineCore pid=2638900)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 2/7 [00:00<00:00, 10.42it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 57%|█████▋ | 4/7 [00:00<00:00, 10.82it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 86%|████████▌ | 6/7 [00:00<00:00, 10.80it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 7/7 [00:00<00:00, 10.91it/s]
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:37 [gpu_model_runner.py:6660] Graph capturing finished in 3 secs, took 0.13 GiB
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:37 [gpu_worker.py:667] CUDA graph pool memory: 0.13 GiB (actual), 0.11 GiB (estimated), difference: 0.02 GiB (12.1%).
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:37 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:38 [core.py:337] init engine (profile, create kv cache, warmup model) took 12.52 s (compilation: 6.02 s)
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:38 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2638900-1783867898685556703.jsonl
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:38 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:38 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:38 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2638188) WARNING 07-12 14:51:38 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8502
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2638188) INFO: Started server process [2638188]
|
||||
(APIServer pid=2638188) INFO: Waiting for application startup.
|
||||
(APIServer pid=2638188) INFO: Application startup complete.
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43588 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638900) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638900) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59190 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59200 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59238 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:52:39 [loggers.py:273] Engine 000: Avg prompt throughput: 5299.4 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:52:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8.5 tokens/s, Avg generation throughput: 95.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 35.1%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:52:59 [loggers.py:273] Engine 000: Avg prompt throughput: 3058.0 tokens/s, Avg generation throughput: 240.8 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.5%, Prefix cache hit rate: 40.3%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5130.2 tokens/s, Avg generation throughput: 333.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 33.3%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38162 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:19 [loggers.py:273] Engine 000: Avg prompt throughput: 3991.7 tokens/s, Avg generation throughput: 168.0 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.7%, Prefix cache hit rate: 29.4%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:29 [loggers.py:273] Engine 000: Avg prompt throughput: 6239.1 tokens/s, Avg generation throughput: 226.6 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.3%, Prefix cache hit rate: 24.2%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33508 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:39 [loggers.py:273] Engine 000: Avg prompt throughput: 7873.3 tokens/s, Avg generation throughput: 414.1 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.4%, Prefix cache hit rate: 21.7%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:49 [loggers.py:273] Engine 000: Avg prompt throughput: 5848.7 tokens/s, Avg generation throughput: 274.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.8%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:59 [loggers.py:273] Engine 000: Avg prompt throughput: 4474.5 tokens/s, Avg generation throughput: 142.1 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.5%, Prefix cache hit rate: 18.4%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36498 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:54:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5897.7 tokens/s, Avg generation throughput: 295.1 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.9%, Prefix cache hit rate: 17.5%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:54:19 [loggers.py:273] Engine 000: Avg prompt throughput: 3893.6 tokens/s, Avg generation throughput: 269.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 17.2%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48862 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:54:29 [loggers.py:273] Engine 000: Avg prompt throughput: 4692.9 tokens/s, Avg generation throughput: 265.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.7%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:54:39 [loggers.py:273] Engine 000: Avg prompt throughput: 8338.1 tokens/s, Avg generation throughput: 294.5 tokens/s, Running: 17 reqs, Waiting: 0 reqs, GPU KV cache usage: 15.7%, Prefix cache hit rate: 16.0%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33368 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp1_mns32/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp1_mns32/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.2421875, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp1_mns32/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp1_mns32/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.025285552,
|
||||
"end_mono_ns": 199530264041528,
|
||||
"end_wall_ns": 1783867957390468387,
|
||||
"start_mono_ns": 199522238755976,
|
||||
"start_wall_ns": 1783867949365182961
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 32,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 28.123532992047,
|
||||
"min": 4.953902173286861,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 682.8775229805615,
|
||||
"min": 74.39436702406965,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.2421875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 137,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.27871102,
|
||||
"end_mono_ns": 199601593594258,
|
||||
"end_wall_ns": 1783868028720022637,
|
||||
"start_mono_ns": 199540314883238,
|
||||
"start_wall_ns": 1783867967441310423
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 137,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "e8e91fd47d9152811ed7bd79e20c0f45ba6677560a419542d9c17abd82bebb4b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"count": 137,
|
||||
"long_gt4096": 47,
|
||||
"offered_req_s": 2.283333333333333,
|
||||
"offered_req_s_per_gpu": 2.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 137,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"raw_length_order_sha256": "93176915562ff9a118f3550157ddd1025d73a6b70cf4d03be9765de9a1b5d744",
|
||||
"request_id_order_sha256": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017"
|
||||
},
|
||||
"slo_pass_count": 137,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 34.5918034883051,
|
||||
"min": 4.451991834606257,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 1472.6779730117414,
|
||||
"min": 33.561496005859226,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.283849775,
|
||||
"end_mono_ns": 199668751241940,
|
||||
"end_wall_ns": 1783868095877669237,
|
||||
"start_mono_ns": 199607467392165,
|
||||
"start_wall_ns": 1783868034593819130
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 48.397922928960206,
|
||||
"min": 4.49719877962177,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1777.0718620158732,
|
||||
"min": 49.70270599005744,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
21
runs/opprof-phase6/phase6/cells/tp1_mns64/cell-valid.json
Normal file
21
runs/opprof-phase6/phase6/cells/tp1_mns64/cell-valid.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"accounting_mode": "A-P6-1-checkpoint-sidecar",
|
||||
"cell": "tp1_mns64",
|
||||
"invariants": {
|
||||
"all_anchor_intervals_covered": true,
|
||||
"all_schema_1": true,
|
||||
"checkpoint_after_all_anchor_intervals": true,
|
||||
"checkpoint_sidecar": true,
|
||||
"checkpoint_within_flush_of_stream": true,
|
||||
"complete_final_newline": true,
|
||||
"encoded_balanced": true,
|
||||
"last_step_matches": true,
|
||||
"no_in_stream_footer": true,
|
||||
"steps_contiguous": true,
|
||||
"two_anchor_intervals": true,
|
||||
"written_matches_records": true,
|
||||
"zero_drops": true
|
||||
},
|
||||
"layer1_records": 9654,
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2638898-1783867899798067669.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp1_mns64/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp1_mns64/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8503 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 64 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.2421875 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/warmup
|
||||
CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.2421875 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/anchor-0.2421875
|
||||
CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.24609375 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/anchor-0.24609375
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638898-1783867899798067669.jsonl","encoded_records":9654,"written_records":9654,"dropped_records":0,"last_step_index":9653,"checkpoint_wall_ns":1783868097891402992,"flush_interval_seconds":1.0,"final":false}
|
||||
437
runs/opprof-phase6/phase6/cells/tp1_mns64/server.log
Normal file
437
runs/opprof-phase6/phase6/cells/tp1_mns64/server.log
Normal file
@@ -0,0 +1,437 @@
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8503, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 64, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:06 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:39471 backend=nccl
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:06 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:01<00:15, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:02<00:14, 1.07s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:03<00:13, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:04<00:13, 1.10s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:05<00:11, 1.08s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:06<00:10, 1.09s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:07<00:09, 1.11s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:08<00:08, 1.09s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:09<00:07, 1.08s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:10<00:06, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:11<00:05, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:12<00:04, 1.04s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:13<00:03, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:15<00:02, 1.09s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:16<00:01, 1.09s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.17it/s]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.03s/it]
|
||||
(EngineCore pid=2638898)
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.52 seconds
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.262335 seconds
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/65b50dadd2/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.30 s
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.131 s
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:31 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/ab53f49ec98f407fe24fecb037cb59739264f283939c70f41986d8369686d472/rank_0_0/model
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:31 [monitor.py:53] torch.compile took 5.83 s in total
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:31 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.21 s
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:32 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=19 (largest=128), FULL=11 (largest=64)
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.19 GiB total
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.33 GiB
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9180 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9220. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 320,304 tokens
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.82x
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2638898) 2026-07-12 14:51:35,044 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2638898) 2026-07-12 14:51:35,116 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2638898)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:01, 9.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:00<00:01, 10.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:00<00:01, 10.01it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:00<00:01, 9.67it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:00<00:01, 9.52it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:01<00:00, 9.72it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:01<00:00, 9.69it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:01<00:00, 9.19it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:01<00:00, 9.33it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:01<00:00, 8.79it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:01<00:00, 8.44it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:01<00:00, 8.20it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:01<00:00, 7.99it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:02<00:00, 8.05it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:02<00:00, 7.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:02<00:00, 8.89it/s]
|
||||
(EngineCore pid=2638898)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:00, 10.99it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:00<00:00, 11.14it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:00<00:00, 11.17it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:00<00:00, 11.23it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:00<00:00, 11.31it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:00<00:00, 11.28it/s]
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:38 [gpu_model_runner.py:6660] Graph capturing finished in 4 secs, took 0.24 GiB
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:38 [gpu_worker.py:667] CUDA graph pool memory: 0.24 GiB (actual), 0.19 GiB (estimated), difference: 0.05 GiB (19.8%).
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:38 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:39 [core.py:337] init engine (profile, create kv cache, warmup model) took 13.60 s (compilation: 5.83 s)
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:39 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2638898-1783867899798067669.jsonl
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:39 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:39 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:39 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2638189) WARNING 07-12 14:51:40 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8503
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2638189) INFO: Started server process [2638189]
|
||||
(APIServer pid=2638189) INFO: Waiting for application startup.
|
||||
(APIServer pid=2638189) INFO: Application startup complete.
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46204 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638898) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638898) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:52:30 [loggers.py:273] Engine 000: Avg prompt throughput: 1668.3 tokens/s, Avg generation throughput: 1.5 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.6%, Prefix cache hit rate: 4.0%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:52:40 [loggers.py:273] Engine 000: Avg prompt throughput: 3630.8 tokens/s, Avg generation throughput: 203.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:52:50 [loggers.py:273] Engine 000: Avg prompt throughput: 11.3 tokens/s, Avg generation throughput: 117.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 42.7%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46612 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:00 [loggers.py:273] Engine 000: Avg prompt throughput: 3820.4 tokens/s, Avg generation throughput: 268.8 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 40.3%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33684 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:10 [loggers.py:273] Engine 000: Avg prompt throughput: 4364.6 tokens/s, Avg generation throughput: 291.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.3%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51684 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51766 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:20 [loggers.py:273] Engine 000: Avg prompt throughput: 4686.8 tokens/s, Avg generation throughput: 194.6 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 28.9%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55574 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55588 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:30 [loggers.py:273] Engine 000: Avg prompt throughput: 7377.9 tokens/s, Avg generation throughput: 242.3 tokens/s, Running: 15 reqs, Waiting: 0 reqs, GPU KV cache usage: 15.3%, Prefix cache hit rate: 24.2%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:40 [loggers.py:273] Engine 000: Avg prompt throughput: 6039.0 tokens/s, Avg generation throughput: 407.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 21.3%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:50 [loggers.py:273] Engine 000: Avg prompt throughput: 5848.5 tokens/s, Avg generation throughput: 230.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.8%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:00 [loggers.py:273] Engine 000: Avg prompt throughput: 4481.1 tokens/s, Avg generation throughput: 175.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.1%, Prefix cache hit rate: 18.4%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:10 [loggers.py:273] Engine 000: Avg prompt throughput: 6653.1 tokens/s, Avg generation throughput: 325.3 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 17.6%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43200 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:20 [loggers.py:273] Engine 000: Avg prompt throughput: 3169.4 tokens/s, Avg generation throughput: 228.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.3%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:30 [loggers.py:273] Engine 000: Avg prompt throughput: 4896.1 tokens/s, Avg generation throughput: 259.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 16.9%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:40 [loggers.py:273] Engine 000: Avg prompt throughput: 8725.1 tokens/s, Avg generation throughput: 309.3 tokens/s, Running: 18 reqs, Waiting: 0 reqs, GPU KV cache usage: 19.8%, Prefix cache hit rate: 15.8%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp1_mns64/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp1_mns64/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.2421875, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp1_mns64/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp1_mns64/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.029548257,
|
||||
"end_mono_ns": 199530257119882,
|
||||
"end_wall_ns": 1783867957383546922,
|
||||
"start_mono_ns": 199522227571625,
|
||||
"start_wall_ns": 1783867949353998820
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 64,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 28.450962196848554,
|
||||
"min": 4.971639937024534,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 697.7603349951096,
|
||||
"min": 52.274041023338214,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.21875, "kind": "anchor", "pass_rate": 0.9917355371900827, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.21875,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 121,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.294286331,
|
||||
"end_mono_ns": 199668659416799,
|
||||
"end_wall_ns": 1783868095785843711,
|
||||
"start_mono_ns": 199607365130468,
|
||||
"start_wall_ns": 1783868034491558052
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 121,
|
||||
"pass_rate": 0.9917355371900827,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "f585e4793a8a6a621dbfe622514743ac316e34e3e93aaa1c6ba5f1eb58e8ff2d",
|
||||
"arrival_s": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"count": 121,
|
||||
"long_gt4096": 42,
|
||||
"offered_req_s": 2.0166666666666666,
|
||||
"offered_req_s_per_gpu": 2.0166666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 114,
|
||||
"finite_n": 121,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"raw_length_order_sha256": "061592e591871368954f4f80ee75cde2cfc399bd719f3f5deaad275e9af72e8b",
|
||||
"request_id_order_sha256": "9b8c177ea00e5e42bd3294328201cb57addfd823913a5721714ce69646d715f4"
|
||||
},
|
||||
"slo_pass_count": 120,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 30.344350779553743,
|
||||
"min": 4.4550072992088525,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 2038.4164949937258,
|
||||
"min": 37.96417900593951,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "anchor", "pass_rate": 0.20634920634920634, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2265625,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 48,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 28.662121521,
|
||||
"end_mono_ns": 199568971776020,
|
||||
"end_wall_ns": 1783867996098203014,
|
||||
"start_mono_ns": 199540309654499,
|
||||
"start_wall_ns": 1783867967436081499
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 126,
|
||||
"pass_rate": 0.20634920634920634,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1b67241583bd6f75b1ad3f6d8c86bf0c815306a28f18ea49b1511fed6c091abe",
|
||||
"arrival_s": {
|
||||
"distinct_n": 126,
|
||||
"finite_n": 126,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"count": 126,
|
||||
"long_gt4096": 44,
|
||||
"offered_req_s": 2.1,
|
||||
"offered_req_s_per_gpu": 2.1,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 119,
|
||||
"finite_n": 126,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"raw_length_order_sha256": "29e3240c20df0cc68172d39aeae84d79908acf564c1b1e2edd447a984e7bdb0e",
|
||||
"request_id_order_sha256": "a05848006335fc9f595432bab793a13d427244886e4f04e51ba74d495453ef90"
|
||||
},
|
||||
"slo_pass_count": 26,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 48,
|
||||
"finite_n": 48,
|
||||
"max": 48.57467018882744,
|
||||
"min": 4.850139889790948,
|
||||
"missing_n": 78,
|
||||
"n": 126
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 48,
|
||||
"finite_n": 48,
|
||||
"max": 7013.455057982355,
|
||||
"min": 39.39470701152459,
|
||||
"missing_n": 78,
|
||||
"n": 126
|
||||
}
|
||||
}
|
||||
21
runs/opprof-phase6/phase6/cells/tp1_mns8/cell-valid.json
Normal file
21
runs/opprof-phase6/phase6/cells/tp1_mns8/cell-valid.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"accounting_mode": "A-P6-1-checkpoint-sidecar",
|
||||
"cell": "tp1_mns8",
|
||||
"invariants": {
|
||||
"all_anchor_intervals_covered": true,
|
||||
"all_schema_1": true,
|
||||
"checkpoint_after_all_anchor_intervals": true,
|
||||
"checkpoint_sidecar": true,
|
||||
"checkpoint_within_flush_of_stream": true,
|
||||
"complete_final_newline": true,
|
||||
"encoded_balanced": true,
|
||||
"last_step_matches": true,
|
||||
"no_in_stream_footer": true,
|
||||
"steps_contiguous": true,
|
||||
"two_anchor_intervals": true,
|
||||
"written_matches_records": true,
|
||||
"zero_drops": true
|
||||
},
|
||||
"layer1_records": 7683,
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2638884-1783867942846550713.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp1_mns8/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp1_mns8/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8500 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 8 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/warmup
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/anchor-0.2265625
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.21875 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/anchor-0.21875
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638884-1783867942846550713.jsonl","encoded_records":7683,"written_records":7683,"dropped_records":0,"last_step_index":7682,"checkpoint_wall_ns":1783868096786407319,"flush_interval_seconds":1.0,"final":false}
|
||||
327
runs/opprof-phase6/phase6/cells/tp1_mns8/server.log
Normal file
327
runs/opprof-phase6/phase6/cells/tp1_mns8/server.log
Normal file
@@ -0,0 +1,327 @@
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8500, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 8, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 16, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:06 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:48769 backend=nccl
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:06 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:01<00:15, 1.06s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:02<00:14, 1.07s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:03<00:13, 1.06s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:04<00:13, 1.10s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:05<00:11, 1.07s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:06<00:10, 1.09s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:07<00:09, 1.11s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:08<00:08, 1.10s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:09<00:07, 1.08s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:10<00:06, 1.07s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:11<00:05, 1.06s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:12<00:04, 1.04s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:13<00:03, 1.06s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:15<00:02, 1.09s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:16<00:01, 1.09s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.18it/s]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.03s/it]
|
||||
(EngineCore pid=2638884)
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.49 seconds
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.240093 seconds
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:35 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/b95eb69a3d/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:35 [backends.py:1148] Dynamo bytecode transform time: 9.16 s
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:50 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:58 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 23.00 s
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:03 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/802e9eed10a34f87d59e9a1acfc3b883a63a5eaeb40312e4832ac27dfbf354e5/rank_0_0/model
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:03 [monitor.py:53] torch.compile took 37.35 s in total
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:03 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:05 [monitor.py:81] Initial profiling/warmup run took 2.19 s
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:12 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:17 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [gpu_worker.py:508] Available KV cache memory: 29.44 GiB
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [kv_cache_utils.py:2146] GPU KV cache size: 321,600 tokens
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.85x
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2638884) 2026-07-12 14:52:18,390 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2638884) 2026-07-12 14:52:18,429 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2638884)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 40%|████ | 2/5 [00:00<00:00, 10.98it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 4/5 [00:01<00:00, 2.57it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 1.77it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 2.12it/s]
|
||||
(EngineCore pid=2638884)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/4 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 50%|█████ | 2/4 [00:00<00:00, 11.61it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 11.88it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 11.84it/s]
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:21 [gpu_model_runner.py:6660] Graph capturing finished in 3 secs, took 0.08 GiB
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:21 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (12.2%).
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:21 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:22 [core.py:337] init engine (profile, create kv cache, warmup model) took 56.63 s (compilation: 37.35 s)
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:22 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2638884-1783867942846550713.jsonl
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:22 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:22 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:22 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2638186) WARNING 07-12 14:52:23 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8500
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2638186) INFO: Started server process [2638186]
|
||||
(APIServer pid=2638186) INFO: Waiting for application startup.
|
||||
(APIServer pid=2638186) INFO: Application startup complete.
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:50438 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:50454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:50460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638884) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:50472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638884) WARNING 07-12 14:52:31 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:33 [loggers.py:273] Engine 000: Avg prompt throughput: 1338.9 tokens/s, Avg generation throughput: 0.9 tokens/s, Running: 8 reqs, Waiting: 1 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:43 [loggers.py:273] Engine 000: Avg prompt throughput: 2863.4 tokens/s, Avg generation throughput: 203.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 5.0%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34572 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:53 [loggers.py:273] Engine 000: Avg prompt throughput: 14.3 tokens/s, Avg generation throughput: 153.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 47.0%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51346 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51422 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:53:03 [loggers.py:273] Engine 000: Avg prompt throughput: 3326.2 tokens/s, Avg generation throughput: 188.1 tokens/s, Running: 7 reqs, Waiting: 8 reqs, GPU KV cache usage: 7.0%, Prefix cache hit rate: 39.8%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42476 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:53:13 [loggers.py:273] Engine 000: Avg prompt throughput: 3070.5 tokens/s, Avg generation throughput: 160.1 tokens/s, Running: 7 reqs, Waiting: 3 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 34.1%
|
||||
(APIServer pid=2638186) INFO 07-12 14:53:23 [loggers.py:273] Engine 000: Avg prompt throughput: 674.1 tokens/s, Avg generation throughput: 112.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.4%
|
||||
(APIServer pid=2638186) INFO 07-12 14:53:33 [loggers.py:273] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.4%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35694 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35656 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:03 [loggers.py:273] Engine 000: Avg prompt throughput: 20.3 tokens/s, Avg generation throughput: 219.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 47.8%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:13 [loggers.py:273] Engine 000: Avg prompt throughput: 21.0 tokens/s, Avg generation throughput: 311.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 58.8%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:23 [loggers.py:273] Engine 000: Avg prompt throughput: 1393.0 tokens/s, Avg generation throughput: 110.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 56.5%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36480 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58042 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5941.7 tokens/s, Avg generation throughput: 269.5 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 48.4%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58138 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58206 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:43 [loggers.py:273] Engine 000: Avg prompt throughput: 8113.3 tokens/s, Avg generation throughput: 349.9 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 41.3%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47602 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47612 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:39082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp1_mns8/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp1_mns8/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "warmup", "pass_rate": 0.3125, "feasible": false}
|
||||
74
runs/opprof-phase6/phase6/cells/tp1_mns8/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp1_mns8/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2265625,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 11.581101349,
|
||||
"end_mono_ns": 199533809459936,
|
||||
"end_wall_ns": 1783867960935887048,
|
||||
"start_mono_ns": 199522228358587,
|
||||
"start_wall_ns": 1783867949354785531
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 8,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 0.3125,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "899b928ee7acd2abc5bf86f6641591111d09f06ec3dbf35a687067b5dfe9ad57",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7.4525000000000095,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 6,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "c99abf7970483c5386d5c34fad5a1b2664c29b8363b5b6e096338ed8e9773d27",
|
||||
"request_id_order_sha256": "9a7cbf2821c1a7b2a498858815a6f31b0d307f2c8a4900d8be107e6a15858964"
|
||||
},
|
||||
"slo_pass_count": 5,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 37.95462277145895,
|
||||
"min": 10.364419787317354,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 4013.1819479865953,
|
||||
"min": 2348.2964480062947,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.4921875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.4921875,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 269,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.339108631,
|
||||
"end_mono_ns": 201223962200316,
|
||||
"end_wall_ns": 1783869651088627469,
|
||||
"start_mono_ns": 201162623091685,
|
||||
"start_wall_ns": 1783869589749518962
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 269,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "ffbc7b8dd324c8e745654110a2412b03ddade891beb2bfe022ffde6502b0184b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"count": 269,
|
||||
"long_gt4096": 95,
|
||||
"offered_req_s": 4.483333333333333,
|
||||
"offered_req_s_per_gpu": 2.2416666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 250,
|
||||
"finite_n": 269,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"raw_length_order_sha256": "bc951350376a487d34d2def78fd32a2286e23bf8a7f481ff578009e12e3f1fb8",
|
||||
"request_id_order_sha256": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c"
|
||||
},
|
||||
"slo_pass_count": 269,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 37.621230566938316,
|
||||
"min": 5.02581280306913,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 1802.1751450141892,
|
||||
"min": 38.09416797594167,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.49609375, "kind": "anchor", "pass_rate": 0.08791208791208792, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 85,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 29.202033933,
|
||||
"end_mono_ns": 201151639670622,
|
||||
"end_wall_ns": 1783869578766097690,
|
||||
"start_mono_ns": 201122437636689,
|
||||
"start_wall_ns": 1783869549564063872
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 273,
|
||||
"pass_rate": 0.08791208791208792,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "41168f4bd02f3ef4e83b7440a3dab7458f08df9c2bbd4452fad076b4a2e0eaed",
|
||||
"arrival_s": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"count": 273,
|
||||
"long_gt4096": 97,
|
||||
"offered_req_s": 4.55,
|
||||
"offered_req_s_per_gpu": 2.275,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 253,
|
||||
"finite_n": 273,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"raw_length_order_sha256": "9e38ed1766e3933dc05a7ced5d0a73fed47ce769800f0d2c10014af0ef823f28",
|
||||
"request_id_order_sha256": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80"
|
||||
},
|
||||
"slo_pass_count": 24,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 85,
|
||||
"finite_n": 85,
|
||||
"max": 56.565095488335864,
|
||||
"min": 5.2603521261925215,
|
||||
"missing_n": 188,
|
||||
"n": 273
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 85,
|
||||
"finite_n": 85,
|
||||
"max": 11585.765292984433,
|
||||
"min": 40.21007599658333,
|
||||
"missing_n": 188,
|
||||
"n": 273
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp2_mns16/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp2_mns16/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns16",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 6313,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/opprof/opprof-v1-dp0-pid2661159-1783869522696890561.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp2_mns16/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp2_mns16/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 40-59,60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8511 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 40-59,60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.49609375 --tp 2 --mns 16 --base-url http://127.0.0.1:8511 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/warmup
|
||||
CLIENT taskset -c 40-59,60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.49609375 --tp 2 --mns 16 --base-url http://127.0.0.1:8511 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/anchor-0.49609375
|
||||
CLIENT taskset -c 40-59,60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.4921875 --tp 2 --mns 16 --base-url http://127.0.0.1:8511 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/anchor-0.4921875
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661159-1783869522696890561.jsonl","encoded_records":6313,"written_records":6313,"dropped_records":0,"last_step_index":6312,"checkpoint_wall_ns":1783869798733332776,"flush_interval_seconds":1.0,"final":true}
|
||||
538
runs/opprof-phase6/phase6/cells/tp2_mns16/server.log
Normal file
538
runs/opprof-phase6/phase6/cells/tp2_mns16/server.log
Normal file
@@ -0,0 +1,538 @@
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8511, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'tensor_parallel_size': 2, 'max_num_batched_tokens': 8192, 'max_num_seqs': 16, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:43 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(EngineCore pid=2661159) INFO 07-12 15:16:53 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': True, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 32, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2661159) WARNING 07-12 15:16:53 [multiproc_executor.py:1063] Reducing Torch parallelism from 40 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
(EngineCore pid=2661159) INFO 07-12 15:16:53 [multiproc_executor.py:140] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.27.132.244 (local), world_size=2, local_world_size=2
|
||||
(Worker pid=2661557) INFO 07-12 15:17:04 [parallel_state.py:1588] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:48921 backend=nccl
|
||||
(Worker pid=2661558) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:48921 backend=nccl
|
||||
(Worker pid=2661557) INFO 07-12 15:17:06 [pynccl.py:113] vLLM is using nccl==2.28.9
|
||||
(Worker pid=2661557) INFO 07-12 15:17:08 [cuda_communicator.py:245] Using ['CUSTOM', 'SYMM_MEM', 'PYNCCL'] all-reduce backends (in dispatch order) for group 'tp:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661557) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['PYNCCL'] all-reduce backends (in dispatch order) for group 'ep:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661557) INFO 07-12 15:17:09 [parallel_state.py:1923] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(Worker pid=2661557) INFO 07-12 15:17:09 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:09 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1275.69 GiB.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:08, 1.71it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:09, 1.55it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:01<00:08, 1.52it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:02<00:08, 1.48it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:03<00:07, 1.47it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:04<00:06, 1.45it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:04<00:06, 1.44it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:05<00:05, 1.45it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:06<00:04, 1.44it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:06<00:04, 1.44it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:07<00:03, 1.43it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:08<00:02, 1.43it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:08<00:02, 1.42it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:09<00:01, 1.42it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:10<00:00, 1.44it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.87it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.52it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:21 [default_loader.py:430] Loading weights took 10.50 seconds
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:21 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:21 [gpu_model_runner.py:5259] Model loading took 28.46 GiB memory and 10.959169 seconds
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:31 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/71caca9a09/rank_0_0/backbone for vLLM's torch.compile
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:31 [backends.py:1148] Dynamo bytecode transform time: 9.70 s
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:31 [flashinfer_all_reduce.py:112] Auto-selected flashinfer allreduce backend: trtllm
|
||||
(Worker_TP0 pid=2661557) /tmp/wjh-opprof-phase2-dash0-20260711/.venv/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
||||
(Worker_TP0 pid=2661557) return func(*args, **kwargs)
|
||||
[rank0]:[W712 15:17:32.191117064 ProcessGroupNCCL.cpp:5188] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:32 [flashinfer_all_reduce.py:152] Initialized FlashInfer Allreduce norm fusion workspace with backend=trtllm
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:56 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:03 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 30.28 s
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:18 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/b1870a24d902681201272cd6a2355f94635aa2439d53d1b45755c3823de597b6/rank_0_0/model
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:18 [monitor.py:53] torch.compile took 56.43 s in total
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:18 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:20 [monitor.py:81] Initial profiling/warmup run took 2.40 s
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:22 [shm_broadcast.py:705] No available shared memory broadcast block found in 60 seconds. This typically happens when some processes are hanging or doing some time-consuming work (e.g. compilation, weight/kv cache quantization).
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:35 [gpu_worker.py:508] Available KV cache memory: 57.35 GiB
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:35 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:35 [kv_cache_utils.py:2146] GPU KV cache size: 1,252,864 tokens
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:35 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 30.59x
|
||||
(Worker_TP1 pid=2661558) 2026-07-12 15:18:35,109 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(Worker_TP0 pid=2661557) 2026-07-12 15:18:35,114 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP1 pid=2661558) 2026-07-12 15:18:35,169 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661557) 2026-07-12 15:18:35,170 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661557)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 14%|█▍ | 1/7 [00:00<00:00, 7.87it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 29%|██▊ | 2/7 [00:00<00:00, 8.09it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 43%|████▎ | 3/7 [00:00<00:00, 8.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 4/7 [00:00<00:00, 8.19it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 71%|███████▏ | 5/7 [00:02<00:01, 1.38it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▌ | 6/7 [00:03<00:00, 1.17it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:04<00:00, 1.09it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:04<00:00, 1.58it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 20%|██ | 1/5 [00:00<00:00, 8.34it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 40%|████ | 2/5 [00:00<00:00, 8.87it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 60%|██████ | 3/5 [00:00<00:00, 9.15it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 80%|████████ | 4/5 [00:00<00:00, 9.30it/s](Worker_TP1 pid=2661558) INFO 07-12 15:18:40 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 9.17it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 9.09it/s]
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:40 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:41 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.01 GiB (14.0%).
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:41 [gpu_model_runner.py:6660] Graph capturing finished in 6 secs, took 0.10 GiB
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:41 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.01 GiB (14.0%).
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:41 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:41 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:41 [core.py:337] init engine (profile, create kv cache, warmup model) took 79.74 s (compilation: 56.45 s)
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:42 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/opprof/opprof-v1-dp0-pid2661159-1783869522696890561.jsonl
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:42 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:42 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:43 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:43 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2660455) WARNING 07-12 15:18:43 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8511
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2660455) INFO: Started server process [2660455]
|
||||
(APIServer pid=2660455) INFO: Waiting for application startup.
|
||||
(APIServer pid=2660455) INFO: Application startup complete.
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:51420 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661557) WARNING 07-12 15:18:54 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661557) WARNING 07-12 15:18:56 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:04 [loggers.py:273] Engine 000: Avg prompt throughput: 5201.1 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.4%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52138 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:14 [loggers.py:273] Engine 000: Avg prompt throughput: 25.9 tokens/s, Avg generation throughput: 193.9 tokens/s, Running: 5 reqs, Waiting: 1 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 47.6%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48492 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:24 [loggers.py:273] Engine 000: Avg prompt throughput: 8173.6 tokens/s, Avg generation throughput: 281.9 tokens/s, Running: 16 reqs, Waiting: 27 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 32.6%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48560 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48572 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:34 [loggers.py:273] Engine 000: Avg prompt throughput: 5964.6 tokens/s, Avg generation throughput: 336.0 tokens/s, Running: 11 reqs, Waiting: 12 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 27.1%
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:44 [loggers.py:273] Engine 000: Avg prompt throughput: 3010.9 tokens/s, Avg generation throughput: 276.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 25.0%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44634 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:54 [loggers.py:273] Engine 000: Avg prompt throughput: 21.7 tokens/s, Avg generation throughput: 239.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 38.8%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44642 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34562 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:04 [loggers.py:273] Engine 000: Avg prompt throughput: 46.9 tokens/s, Avg generation throughput: 670.8 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 56.6%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34602 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34694 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:14 [loggers.py:273] Engine 000: Avg prompt throughput: 8386.5 tokens/s, Avg generation throughput: 478.4 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 50.9%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33862 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33892 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:24 [loggers.py:273] Engine 000: Avg prompt throughput: 12462.8 tokens/s, Avg generation throughput: 461.2 tokens/s, Running: 16 reqs, Waiting: 1 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 44.1%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58138 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58172 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:34 [loggers.py:273] Engine 000: Avg prompt throughput: 10574.6 tokens/s, Avg generation throughput: 559.7 tokens/s, Running: 16 reqs, Waiting: 8 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 39.5%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58508 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34152 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:44 [loggers.py:273] Engine 000: Avg prompt throughput: 14020.4 tokens/s, Avg generation throughput: 680.9 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 35.0%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp2_mns16/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns16/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.49609375, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp2_mns16/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp2_mns16/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 5.344845622,
|
||||
"end_mono_ns": 201112093538719,
|
||||
"end_wall_ns": 1783869539219965703,
|
||||
"start_mono_ns": 201106748693097,
|
||||
"start_wall_ns": 1783869533875120325
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 16,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "818d316f68acc6502ed33a877bf88c2ba9b57a1ff6d6780621db0d30eddf3494",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3.4938000000000105,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 7,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "2eb3b78b31efc9354c64c8caa95f08353330553d1ab745a8e0ceab038a3a3d5e",
|
||||
"request_id_order_sha256": "0009512d07ec4c5bb2dd1283c767967a6dfd5bc4c7b7d65fb26169d39a98832e"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 32.09012237012636,
|
||||
"min": 9.460583771716466,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 1483.7540120061021,
|
||||
"min": 370.18870600149967,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75, "kind": "anchor", "pass_rate": 0.4117647058823529, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 221,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 38.257993674,
|
||||
"end_mono_ns": 201200786085530,
|
||||
"end_wall_ns": 1783869627912512603,
|
||||
"start_mono_ns": 201162528091856,
|
||||
"start_wall_ns": 1783869589654519727
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 391,
|
||||
"pass_rate": 0.4117647058823529,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "d3ed714128e985d0e0a0ca6b3767210c181114ab6694fb066a60aecf5afa9f9a",
|
||||
"arrival_s": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"count": 391,
|
||||
"long_gt4096": 140,
|
||||
"offered_req_s": 6.516666666666667,
|
||||
"offered_req_s_per_gpu": 3.2583333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 361,
|
||||
"finite_n": 391,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"raw_length_order_sha256": "8152bbaf9f27e84e6fbd6359174d17de4b9c96ea8b8c58e3760290be3469b2a9",
|
||||
"request_id_order_sha256": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c"
|
||||
},
|
||||
"slo_pass_count": 161,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 221,
|
||||
"finite_n": 221,
|
||||
"max": 59.45439488186105,
|
||||
"min": 6.009525047224675,
|
||||
"missing_n": 170,
|
||||
"n": 391
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 221,
|
||||
"finite_n": 221,
|
||||
"max": 3492.7126489928924,
|
||||
"min": 39.199416001793,
|
||||
"missing_n": 170,
|
||||
"n": 391
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75390625, "kind": "anchor", "pass_rate": 0.027918781725888325, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75390625,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 110,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 27.144585398,
|
||||
"end_mono_ns": 201149546363473,
|
||||
"end_wall_ns": 1783869576672790977,
|
||||
"start_mono_ns": 201122401778075,
|
||||
"start_wall_ns": 1783869549528204982
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 394,
|
||||
"pass_rate": 0.027918781725888325,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "60fc43518e79fd46abe59afde67663a762ca17e817202fa52803040ac9056796",
|
||||
"arrival_s": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"count": 394,
|
||||
"long_gt4096": 142,
|
||||
"offered_req_s": 6.566666666666666,
|
||||
"offered_req_s_per_gpu": 3.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 364,
|
||||
"finite_n": 394,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"raw_length_order_sha256": "4a9cc03e351cce8c558b30732418c878fc3b6d24e64bef6717ce7c87c384384f",
|
||||
"request_id_order_sha256": "2677a6a12ff0e104d6246dd121d0667ff4fe3d25f8d68699547906420246ebdd"
|
||||
},
|
||||
"slo_pass_count": 11,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 110,
|
||||
"finite_n": 110,
|
||||
"max": 85.24905312595537,
|
||||
"min": 5.966831590589796,
|
||||
"missing_n": 284,
|
||||
"n": 394
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 110,
|
||||
"finite_n": 110,
|
||||
"max": 8747.10591501207,
|
||||
"min": 43.805207998957485,
|
||||
"missing_n": 284,
|
||||
"n": 394
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp2_mns32/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp2_mns32/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns32",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 17712,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/opprof/opprof-v1-dp0-pid2661056-1783869524249984543.jsonl"
|
||||
}
|
||||
6
runs/opprof-phase6/phase6/cells/tp2_mns32/commands.log
Normal file
6
runs/opprof-phase6/phase6/cells/tp2_mns32/commands.log
Normal file
@@ -0,0 +1,6 @@
|
||||
SERVER taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8512 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 32 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/warmup
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/anchor-0.75390625
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/anchor-0.75
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/confirm-1-anchor-0.75390625
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/confirm-2-anchor-0.75
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75390625, "kind": "anchor", "pass_rate": 0.9568527918781726, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75390625,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 394,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 64.605127417,
|
||||
"end_mono_ns": 201294724925354,
|
||||
"end_wall_ns": 1783869721851353202,
|
||||
"start_mono_ns": 201230119797937,
|
||||
"start_wall_ns": 1783869657246225290
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 394,
|
||||
"pass_rate": 0.9568527918781726,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "60fc43518e79fd46abe59afde67663a762ca17e817202fa52803040ac9056796",
|
||||
"arrival_s": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"count": 394,
|
||||
"long_gt4096": 142,
|
||||
"offered_req_s": 6.566666666666666,
|
||||
"offered_req_s_per_gpu": 3.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 364,
|
||||
"finite_n": 394,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"raw_length_order_sha256": "4a9cc03e351cce8c558b30732418c878fc3b6d24e64bef6717ce7c87c384384f",
|
||||
"request_id_order_sha256": "2677a6a12ff0e104d6246dd121d0667ff4fe3d25f8d68699547906420246ebdd"
|
||||
},
|
||||
"slo_pass_count": 377,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 50.88291912592625,
|
||||
"min": 5.975462165367826,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 3372.0373090181965,
|
||||
"min": 39.65454100398347,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 391,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 60.782576125,
|
||||
"end_mono_ns": 201361364171726,
|
||||
"end_wall_ns": 1783869788490598727,
|
||||
"start_mono_ns": 201300581595601,
|
||||
"start_wall_ns": 1783869727708022566
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 391,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "d3ed714128e985d0e0a0ca6b3767210c181114ab6694fb066a60aecf5afa9f9a",
|
||||
"arrival_s": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"count": 391,
|
||||
"long_gt4096": 140,
|
||||
"offered_req_s": 6.516666666666667,
|
||||
"offered_req_s_per_gpu": 3.2583333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 361,
|
||||
"finite_n": 391,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"raw_length_order_sha256": "8152bbaf9f27e84e6fbd6359174d17de4b9c96ea8b8c58e3760290be3469b2a9",
|
||||
"request_id_order_sha256": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c"
|
||||
},
|
||||
"slo_pass_count": 391,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 9.731004889788634,
|
||||
"min": 5.9432724644759976,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 81.50044700596482,
|
||||
"min": 39.63042498799041,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661056-1783869524249984543.jsonl","encoded_records":17712,"written_records":17712,"dropped_records":0,"last_step_index":17711,"checkpoint_wall_ns":1783869804704938995,"flush_interval_seconds":1.0,"final":true}
|
||||
1313
runs/opprof-phase6/phase6/cells/tp2_mns32/server.log
Normal file
1313
runs/opprof-phase6/phase6/cells/tp2_mns32/server.log
Normal file
File diff suppressed because it is too large
Load Diff
1
runs/opprof-phase6/phase6/cells/tp2_mns32/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns32/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75390625, "kind": "warmup", "pass_rate": 0.75, "feasible": false}
|
||||
74
runs/opprof-phase6/phase6/cells/tp2_mns32/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp2_mns32/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75390625,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 7.481165294,
|
||||
"end_mono_ns": 201114243480033,
|
||||
"end_wall_ns": 1783869541369906988,
|
||||
"start_mono_ns": 201106762314739,
|
||||
"start_wall_ns": 1783869533888742060
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 32,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 0.75,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "440dc3774dd5b89ed800eee12e44a20f62b0ee9b66094d8e05fa343fb2ba3dd7",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 2.4043000000000125,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 5,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7233.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "cb99e1339d84161d25fbc6a3c6ed983f34c0276e7712a62c176a8b92b8cb7b93",
|
||||
"request_id_order_sha256": "9d18ed5ddc01d283277e84d107b444833f6ab2838675a8c7ffaedb7a0c1d50cf"
|
||||
},
|
||||
"slo_pass_count": 12,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 48.713730740283914,
|
||||
"min": 9.619937283601098,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3962.258840998402,
|
||||
"min": 814.5189289934933,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
1
runs/opprof-phase6/phase6/cells/tp2_mns64/anchor-0.5.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns64/anchor-0.5.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns64", "anchor": 0.5, "kind": "anchor", "pass_rate": 0.4601449275362319, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.5,
|
||||
"cell": "tp2_mns64",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 142,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 32.625174607,
|
||||
"end_mono_ns": 201195153573715,
|
||||
"end_wall_ns": 1783869622280001349,
|
||||
"start_mono_ns": 201162528399108,
|
||||
"start_wall_ns": 1783869589654826388
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 276,
|
||||
"pass_rate": 0.4601449275362319,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
|
||||
"arrival_s": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"count": 276,
|
||||
"long_gt4096": 99,
|
||||
"offered_req_s": 4.6,
|
||||
"offered_req_s_per_gpu": 2.3,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 256,
|
||||
"finite_n": 276,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
|
||||
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
|
||||
},
|
||||
"slo_pass_count": 127,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 142,
|
||||
"finite_n": 142,
|
||||
"max": 65.60344734654475,
|
||||
"min": 4.993453378033802,
|
||||
"missing_n": 134,
|
||||
"n": 276
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 142,
|
||||
"finite_n": 142,
|
||||
"max": 1354.7493759833742,
|
||||
"min": 39.566546998685226,
|
||||
"missing_n": 134,
|
||||
"n": 276
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns64", "anchor": 0.75, "kind": "anchor", "pass_rate": 0.14066496163682865, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns64",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 129,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 26.447176041,
|
||||
"end_mono_ns": 201148848516622,
|
||||
"end_wall_ns": 1783869575974943780,
|
||||
"start_mono_ns": 201122401340581,
|
||||
"start_wall_ns": 1783869549527767999
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 391,
|
||||
"pass_rate": 0.14066496163682865,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "d3ed714128e985d0e0a0ca6b3767210c181114ab6694fb066a60aecf5afa9f9a",
|
||||
"arrival_s": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"count": 391,
|
||||
"long_gt4096": 140,
|
||||
"offered_req_s": 6.516666666666667,
|
||||
"offered_req_s_per_gpu": 3.2583333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 361,
|
||||
"finite_n": 391,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"raw_length_order_sha256": "8152bbaf9f27e84e6fbd6359174d17de4b9c96ea8b8c58e3760290be3469b2a9",
|
||||
"request_id_order_sha256": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c"
|
||||
},
|
||||
"slo_pass_count": 55,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 129,
|
||||
"max": 109.8500202991083,
|
||||
"min": 5.983953999894429,
|
||||
"missing_n": 262,
|
||||
"n": 391
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 129,
|
||||
"max": 2141.4210819930304,
|
||||
"min": 43.8292009930592,
|
||||
"missing_n": 262,
|
||||
"n": 391
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp2_mns64/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp2_mns64/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns64",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 4537,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/opprof/opprof-v1-dp0-pid2661205-1783869525995600880.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp2_mns64/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp2_mns64/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8513 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 64 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns64 --anchor 0.75 --tp 2 --mns 64 --base-url http://127.0.0.1:8513 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/warmup
|
||||
CLIENT taskset -c 120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns64 --anchor 0.75 --tp 2 --mns 64 --base-url http://127.0.0.1:8513 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/anchor-0.75
|
||||
CLIENT taskset -c 120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns64 --anchor 0.5 --tp 2 --mns 64 --base-url http://127.0.0.1:8513 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/anchor-0.5
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661205-1783869525995600880.jsonl","encoded_records":4537,"written_records":4537,"dropped_records":0,"last_step_index":4536,"checkpoint_wall_ns":1783869810430748669,"flush_interval_seconds":1.0,"final":true}
|
||||
453
runs/opprof-phase6/phase6/cells/tp2_mns64/server.log
Normal file
453
runs/opprof-phase6/phase6/cells/tp2_mns64/server.log
Normal file
@@ -0,0 +1,453 @@
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8513, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'tensor_parallel_size': 2, 'max_num_batched_tokens': 8192, 'max_num_seqs': 64, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:44 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(EngineCore pid=2661205) INFO 07-12 15:16:54 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': True, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2661205) WARNING 07-12 15:16:54 [multiproc_executor.py:1063] Reducing Torch parallelism from 40 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
(EngineCore pid=2661205) INFO 07-12 15:16:54 [multiproc_executor.py:140] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.27.132.244 (local), world_size=2, local_world_size=2
|
||||
(Worker pid=2661569) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:41317 backend=nccl
|
||||
(Worker pid=2661570) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:41317 backend=nccl
|
||||
(Worker pid=2661569) INFO 07-12 15:17:06 [pynccl.py:113] vLLM is using nccl==2.28.9
|
||||
(Worker pid=2661569) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['CUSTOM', 'SYMM_MEM', 'PYNCCL'] all-reduce backends (in dispatch order) for group 'tp:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661569) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['PYNCCL'] all-reduce backends (in dispatch order) for group 'ep:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661569) INFO 07-12 15:17:09 [parallel_state.py:1923] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(Worker pid=2661569) INFO 07-12 15:17:10 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:10 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:10 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:10 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:10 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:11 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1275.25 GiB.
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:11 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:09, 1.58it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:09, 1.40it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:02<00:09, 1.36it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:02<00:08, 1.36it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:03<00:08, 1.35it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:04<00:07, 1.36it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:05<00:06, 1.36it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:05<00:05, 1.35it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:06<00:05, 1.33it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:07<00:04, 1.33it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:08<00:03, 1.32it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:08<00:03, 1.30it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:09<00:02, 1.32it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:10<00:01, 1.33it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:11<00:00, 1.34it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:11<00:00, 1.74it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:11<00:00, 1.41it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:22 [default_loader.py:430] Loading weights took 11.35 seconds
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:22 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:23 [gpu_model_runner.py:5259] Model loading took 28.46 GiB memory and 11.752905 seconds
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:32 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/7a9372c548/rank_0_0/backbone for vLLM's torch.compile
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:32 [backends.py:1148] Dynamo bytecode transform time: 9.56 s
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:32 [flashinfer_all_reduce.py:112] Auto-selected flashinfer allreduce backend: trtllm
|
||||
(Worker_TP0 pid=2661569) /tmp/wjh-opprof-phase2-dash0-20260711/.venv/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
||||
(Worker_TP0 pid=2661569) return func(*args, **kwargs)
|
||||
[rank0]:[W712 15:17:33.273268179 ProcessGroupNCCL.cpp:5188] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:33 [flashinfer_all_reduce.py:152] Initialized FlashInfer Allreduce norm fusion workspace with backend=trtllm
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:57 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:04 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 30.15 s
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:18 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/7c6e0ddc6dcbaf895fc62cc90d8a628d2dfd8c5b38e2d7747d4762b6a26b182b/rank_0_0/model
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:18 [monitor.py:53] torch.compile took 55.50 s in total
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:19 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:21 [monitor.py:81] Initial profiling/warmup run took 2.55 s
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:24 [shm_broadcast.py:705] No available shared memory broadcast block found in 60 seconds. This typically happens when some processes are hanging or doing some time-consuming work (e.g. compilation, weight/kv cache quantization).
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=19 (largest=128), FULL=11 (largest=64)
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=19 (largest=128), FULL=11 (largest=64)
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:32 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:32 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:33 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.17 GiB total
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:33 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.17 GiB total
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9182 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9218. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:34 [gpu_worker.py:508] Available KV cache memory: 57.27 GiB
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9182 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9218. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:34 [kv_cache_utils.py:2146] GPU KV cache size: 1,251,040 tokens
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 30.54x
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:34 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:34 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(Worker_TP1 pid=2661570) 2026-07-12 15:18:34,433 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP0 pid=2661569) 2026-07-12 15:18:34,434 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP1 pid=2661570) 2026-07-12 15:18:34,488 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661569) 2026-07-12 15:18:34,489 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661569)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:02, 8.64it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:00<00:01, 8.66it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:00<00:01, 8.61it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:00<00:01, 8.68it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:00<00:01, 8.60it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:00<00:01, 8.62it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:00<00:01, 8.61it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:00<00:01, 8.62it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:01<00:01, 8.67it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:01<00:01, 8.67it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:01<00:00, 8.59it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:02<00:03, 2.22it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:02<00:02, 2.84it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:03<00:02, 1.71it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:03<00:01, 2.24it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:03<00:01, 2.87it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:05<00:01, 1.37it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:06<00:00, 1.25it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 1.15it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 2.50it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:01, 9.49it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:00, 9.41it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:00<00:00, 9.08it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:00<00:00, 9.02it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:00<00:00, 8.97it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:00<00:00, 8.15it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:00<00:00, 8.52it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:00<00:00, 8.81it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:01<00:00, 8.94it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:01<00:00, 9.03it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:01<00:00, 9.16it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:01<00:00, 8.94it/s]
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:43 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:43 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:44 [gpu_model_runner.py:6660] Graph capturing finished in 10 secs, took 0.20 GiB
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:44 [gpu_worker.py:667] CUDA graph pool memory: 0.2 GiB (actual), 0.17 GiB (estimated), difference: 0.03 GiB (14.9%).
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:44 [gpu_worker.py:667] CUDA graph pool memory: 0.2 GiB (actual), 0.17 GiB (estimated), difference: 0.03 GiB (14.9%).
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:44 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:44 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:44 [core.py:337] init engine (profile, create kv cache, warmup model) took 81.68 s (compilation: 55.67 s)
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:45 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/opprof/opprof-v1-dp0-pid2661205-1783869525995600880.jsonl
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:46 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:46 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:47 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2660457) WARNING 07-12 15:18:47 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8513
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2660457) INFO: Started server process [2660457]
|
||||
(APIServer pid=2660457) INFO: Waiting for application startup.
|
||||
(APIServer pid=2660457) INFO: Application startup complete.
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:57528 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661569) WARNING 07-12 15:18:54 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55676 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55686 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661569) WARNING 07-12 15:18:56 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:57 [loggers.py:273] Engine 000: Avg prompt throughput: 3062.3 tokens/s, Avg generation throughput: 3.4 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 7.2%
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:07 [loggers.py:273] Engine 000: Avg prompt throughput: 922.8 tokens/s, Avg generation throughput: 200.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 6.5%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50296 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47686 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:17 [loggers.py:273] Engine 000: Avg prompt throughput: 7057.2 tokens/s, Avg generation throughput: 208.8 tokens/s, Running: 28 reqs, Waiting: 8 reqs, GPU KV cache usage: 6.9%, Prefix cache hit rate: 30.7%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48088 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:27 [loggers.py:273] Engine 000: Avg prompt throughput: 13039.6 tokens/s, Avg generation throughput: 707.1 tokens/s, Running: 63 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.9%, Prefix cache hit rate: 21.3%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43480 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:37 [loggers.py:273] Engine 000: Avg prompt throughput: 13455.8 tokens/s, Avg generation throughput: 735.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.6%
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:47 [loggers.py:273] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.6%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:57 [loggers.py:273] Engine 000: Avg prompt throughput: 36.2 tokens/s, Avg generation throughput: 441.2 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 33.0%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49346 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:20:07 [loggers.py:273] Engine 000: Avg prompt throughput: 45.9 tokens/s, Avg generation throughput: 672.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 46.1%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49422 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53754 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53766 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53846 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp2_mns64/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns64/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns64", "anchor": 0.75, "kind": "warmup", "pass_rate": 0.75, "feasible": false}
|
||||
74
runs/opprof-phase6/phase6/cells/tp2_mns64/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp2_mns64/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 7.415686836,
|
||||
"end_mono_ns": 201114188199246,
|
||||
"end_wall_ns": 1783869541314626167,
|
||||
"start_mono_ns": 201106772512410,
|
||||
"start_wall_ns": 1783869533898939505
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 64,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 0.75,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "97f6ded38afa2a0fa9729d0553429b3df62e3d4e99710f4c19a706b1eba8fdaa",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 2.723000000000002,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 5,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7233.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "76f4ce7bc309a1108d5d4b24ba24b374c14483a3dd35cd5648fbeafca60221ba",
|
||||
"request_id_order_sha256": "b38587f14f8c188d131b4c5c8411951ec571999fb2ac6be6517f1ad952208852"
|
||||
},
|
||||
"slo_pass_count": 12,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 48.44081654317868,
|
||||
"min": 9.34166979529587,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3932.37450599554,
|
||||
"min": 783.6844140256289,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns8", "anchor": 0.4921875, "kind": "anchor", "pass_rate": 0.6914498141263941, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.4921875,
|
||||
"cell": "tp2_mns8",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 216,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 51.56724132,
|
||||
"end_mono_ns": 201214250324754,
|
||||
"end_wall_ns": 1783869641376751823,
|
||||
"start_mono_ns": 201162683083434,
|
||||
"start_wall_ns": 1783869589809510519
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 269,
|
||||
"pass_rate": 0.6914498141263941,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "ffbc7b8dd324c8e745654110a2412b03ddade891beb2bfe022ffde6502b0184b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"count": 269,
|
||||
"long_gt4096": 95,
|
||||
"offered_req_s": 4.483333333333333,
|
||||
"offered_req_s_per_gpu": 2.2416666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 250,
|
||||
"finite_n": 269,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"raw_length_order_sha256": "bc951350376a487d34d2def78fd32a2286e23bf8a7f481ff578009e12e3f1fb8",
|
||||
"request_id_order_sha256": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c"
|
||||
},
|
||||
"slo_pass_count": 186,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 216,
|
||||
"finite_n": 216,
|
||||
"max": 22.936211125952685,
|
||||
"min": 5.354799519524008,
|
||||
"missing_n": 53,
|
||||
"n": 269
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 216,
|
||||
"finite_n": 216,
|
||||
"max": 3692.623541021021,
|
||||
"min": 40.143018995877355,
|
||||
"missing_n": 53,
|
||||
"n": 269
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns8", "anchor": 0.49609375, "kind": "anchor", "pass_rate": 0.09157509157509157, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns8",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 87,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 33.978401963,
|
||||
"end_mono_ns": 201156432397399,
|
||||
"end_wall_ns": 1783869583558824468,
|
||||
"start_mono_ns": 201122453995436,
|
||||
"start_wall_ns": 1783869549580422810
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 273,
|
||||
"pass_rate": 0.09157509157509157,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "41168f4bd02f3ef4e83b7440a3dab7458f08df9c2bbd4452fad076b4a2e0eaed",
|
||||
"arrival_s": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"count": 273,
|
||||
"long_gt4096": 97,
|
||||
"offered_req_s": 4.55,
|
||||
"offered_req_s_per_gpu": 2.275,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 253,
|
||||
"finite_n": 273,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"raw_length_order_sha256": "9e38ed1766e3933dc05a7ced5d0a73fed47ce769800f0d2c10014af0ef823f28",
|
||||
"request_id_order_sha256": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80"
|
||||
},
|
||||
"slo_pass_count": 25,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 87,
|
||||
"finite_n": 87,
|
||||
"max": 38.90758789764116,
|
||||
"min": 6.895991826913957,
|
||||
"missing_n": 186,
|
||||
"n": 273
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 87,
|
||||
"finite_n": 87,
|
||||
"max": 16124.392641999293,
|
||||
"min": 52.06192599143833,
|
||||
"missing_n": 186,
|
||||
"n": 273
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp2_mns8/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp2_mns8/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns8",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 6687,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/opprof/opprof-v1-dp0-pid2661256-1783869521184806299.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp2_mns8/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp2_mns8/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8510 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 8 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.49609375 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/warmup
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.49609375 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/anchor-0.49609375
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.4921875 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/anchor-0.4921875
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661256-1783869521184806299.jsonl","encoded_records":6687,"written_records":6687,"dropped_records":0,"last_step_index":6686,"checkpoint_wall_ns":1783869792959224277,"flush_interval_seconds":1.0,"final":true}
|
||||
486
runs/opprof-phase6/phase6/cells/tp2_mns8/server.log
Normal file
486
runs/opprof-phase6/phase6/cells/tp2_mns8/server.log
Normal file
@@ -0,0 +1,486 @@
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8510, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'tensor_parallel_size': 2, 'max_num_batched_tokens': 8192, 'max_num_seqs': 8, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:45 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(EngineCore pid=2661256) INFO 07-12 15:16:55 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': True, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 16, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2661256) WARNING 07-12 15:16:55 [multiproc_executor.py:1063] Reducing Torch parallelism from 40 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
(EngineCore pid=2661256) INFO 07-12 15:16:55 [multiproc_executor.py:140] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.27.132.244 (local), world_size=2, local_world_size=2
|
||||
(Worker pid=2661580) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:42171 backend=nccl
|
||||
(Worker pid=2661581) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:42171 backend=nccl
|
||||
(Worker pid=2661580) INFO 07-12 15:17:06 [pynccl.py:113] vLLM is using nccl==2.28.9
|
||||
(Worker pid=2661580) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['CUSTOM', 'SYMM_MEM', 'PYNCCL'] all-reduce backends (in dispatch order) for group 'tp:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661580) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['PYNCCL'] all-reduce backends (in dispatch order) for group 'ep:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661580) INFO 07-12 15:17:09 [parallel_state.py:1923] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(Worker pid=2661580) INFO 07-12 15:17:10 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1275.26 GiB.
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:09, 1.61it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:09, 1.53it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:01<00:08, 1.51it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:02<00:08, 1.47it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:03<00:07, 1.45it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:04<00:06, 1.43it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:04<00:06, 1.44it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:05<00:05, 1.45it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:06<00:04, 1.44it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:06<00:04, 1.44it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:07<00:03, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:08<00:02, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:08<00:02, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:09<00:01, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:10<00:00, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.89it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.53it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:21 [default_loader.py:430] Loading weights took 10.45 seconds
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:21 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:22 [gpu_model_runner.py:5259] Model loading took 28.46 GiB memory and 10.837326 seconds
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/0c37cd99d7/rank_0_0/backbone for vLLM's torch.compile
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [backends.py:1148] Dynamo bytecode transform time: 9.74 s
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [flashinfer_all_reduce.py:112] Auto-selected flashinfer allreduce backend: trtllm
|
||||
(Worker_TP0 pid=2661580) /tmp/wjh-opprof-phase2-dash0-20260711/.venv/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
||||
(Worker_TP0 pid=2661580) return func(*args, **kwargs)
|
||||
[rank0]:[W712 15:17:32.753967979 ProcessGroupNCCL.cpp:5188] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:33 [flashinfer_all_reduce.py:152] Initialized FlashInfer Allreduce norm fusion workspace with backend=trtllm
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:56 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:03 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 30.05 s
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/7db77e4784ad115c73ccf5b7472a39190ea4c85a91f415675a26d40f2c108d60/rank_0_0/model
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [monitor.py:53] torch.compile took 56.36 s in total
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:21 [monitor.py:81] Initial profiling/warmup run took 2.52 s
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:23 [shm_broadcast.py:705] No available shared memory broadcast block found in 60 seconds. This typically happens when some processes are hanging or doing some time-consuming work (e.g. compilation, weight/kv cache quantization).
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [gpu_worker.py:508] Available KV cache memory: 57.37 GiB
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:35 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:35 [kv_cache_utils.py:2146] GPU KV cache size: 1,253,168 tokens
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:35 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 30.59x
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(Worker_TP0 pid=2661580) 2026-07-12 15:18:35,188 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP1 pid=2661581) 2026-07-12 15:18:35,200 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP0 pid=2661580) 2026-07-12 15:18:35,244 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP1 pid=2661581) 2026-07-12 15:18:35,270 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661580)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 20%|██ | 1/5 [00:00<00:00, 8.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 40%|████ | 2/5 [00:00<00:00, 8.50it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 60%|██████ | 3/5 [00:00<00:00, 8.72it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 4/5 [00:01<00:00, 1.89it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 1.37it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 1.93it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/4 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 25%|██▌ | 1/4 [00:00<00:00, 8.58it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 50%|█████ | 2/4 [00:00<00:00, 9.08it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 75%|███████▌ | 3/4 [00:00<00:00, 9.18it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 9.44it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 9.28it/s]
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:38 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:38 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:39 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (14.3%).
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [gpu_model_runner.py:6660] Graph capturing finished in 4 secs, took 0.08 GiB
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (14.3%).
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:39 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:39 [core.py:337] init engine (profile, create kv cache, warmup model) took 77.56 s (compilation: 56.36 s)
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:41 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/opprof/opprof-v1-dp0-pid2661256-1783869521184806299.jsonl
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:41 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:41 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:42 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:42 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2660454) WARNING 07-12 15:18:42 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:42 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8510
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2660454) INFO: Started server process [2660454]
|
||||
(APIServer pid=2660454) INFO: Waiting for application startup.
|
||||
(APIServer pid=2660454) INFO: Application startup complete.
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:34364 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661580) WARNING 07-12 15:18:54 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661580) WARNING 07-12 15:18:57 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:03 [loggers.py:273] Engine 000: Avg prompt throughput: 5201.2 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.4%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:13 [loggers.py:273] Engine 000: Avg prompt throughput: 11.8 tokens/s, Avg generation throughput: 110.7 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 50.0%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42094 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42190 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42296 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:23 [loggers.py:273] Engine 000: Avg prompt throughput: 5066.8 tokens/s, Avg generation throughput: 298.5 tokens/s, Running: 8 reqs, Waiting: 33 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 36.8%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42368 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5845.7 tokens/s, Avg generation throughput: 315.7 tokens/s, Running: 7 reqs, Waiting: 26 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 29.2%
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:43 [loggers.py:273] Engine 000: Avg prompt throughput: 6481.6 tokens/s, Avg generation throughput: 371.0 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 24.8%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:53 [loggers.py:273] Engine 000: Avg prompt throughput: 16.3 tokens/s, Avg generation throughput: 193.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 35.5%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48154 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48172 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35846 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:20:03 [loggers.py:273] Engine 000: Avg prompt throughput: 45.0 tokens/s, Avg generation throughput: 646.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 54.1%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:20:13 [loggers.py:273] Engine 000: Avg prompt throughput: 6622.6 tokens/s, Avg generation throughput: 499.2 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 52.5%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:20:23 [loggers.py:273] Engine 000: Avg prompt throughput: 10999.7 tokens/s, Avg generation throughput: 475.5 tokens/s, Running: 8 reqs, Waiting: 8 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 45.6%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57844 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57864 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57892 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:20:33 [loggers.py:273] Engine 000: Avg prompt throughput: 10684.1 tokens/s, Avg generation throughput: 487.2 tokens/s, Running: 8 reqs, Waiting: 14 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 40.8%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33134 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns8", "anchor": 0.49609375, "kind": "warmup", "pass_rate": 0.625, "feasible": false}
|
||||
74
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 9.026111125,
|
||||
"end_mono_ns": 201115775651265,
|
||||
"end_wall_ns": 1783869542902078194,
|
||||
"start_mono_ns": 201106749540140,
|
||||
"start_wall_ns": 1783869533875967269
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 8,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 0.625,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "818d316f68acc6502ed33a877bf88c2ba9b57a1ff6d6780621db0d30eddf3494",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3.4938000000000105,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 7,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "2eb3b78b31efc9354c64c8caa95f08353330553d1ab745a8e0ceab038a3a3d5e",
|
||||
"request_id_order_sha256": "0009512d07ec4c5bb2dd1283c767967a6dfd5bc4c7b7d65fb26169d39a98832e"
|
||||
},
|
||||
"slo_pass_count": 10,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 39.46136329144306,
|
||||
"min": 7.347993740140952,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 4594.941660994664,
|
||||
"min": 762.8166690119542,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.033182214016, "kind": "anchor", "pass_rate": 0.6086956521739131, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.033182214016,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 413,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 49.259978895,
|
||||
"end_mono_ns": 201650764016357,
|
||||
"end_wall_ns": 1783870077890443949,
|
||||
"start_mono_ns": 201601504037462,
|
||||
"start_wall_ns": 1783870028630464875
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 575,
|
||||
"pass_rate": 0.6086956521739131,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "86ee457cd91595c76454d54ebc69890836ef603ceb96deaf9c7cf05e91de9407",
|
||||
"arrival_s": {
|
||||
"distinct_n": 574,
|
||||
"finite_n": 575,
|
||||
"max": 59.90990000000002,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 575
|
||||
},
|
||||
"count": 575,
|
||||
"long_gt4096": 178,
|
||||
"offered_req_s": 9.583333333333334,
|
||||
"offered_req_s_per_gpu": 2.3958333333333335,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 524,
|
||||
"finite_n": 575,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 575
|
||||
},
|
||||
"raw_length_order_sha256": "8354300b1680841ab43ef8164307d7c1f28bc877994b8d1e7584aff030677560",
|
||||
"request_id_order_sha256": "e6b624a0212807df3733c54b4a2160fba24f19ca4776b14f34918d83428bba7f"
|
||||
},
|
||||
"slo_pass_count": 350,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 413,
|
||||
"finite_n": 413,
|
||||
"max": 16.932683684963983,
|
||||
"min": 4.637226771506459,
|
||||
"missing_n": 162,
|
||||
"n": 575
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 413,
|
||||
"finite_n": 413,
|
||||
"max": 4166.508020018227,
|
||||
"min": 38.04149298230186,
|
||||
"missing_n": 162,
|
||||
"n": 575
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.033717411016, "kind": "anchor", "pass_rate": 0.03583617747440273, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.033717411016,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 199,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 41.949754618,
|
||||
"end_mono_ns": 201595581507061,
|
||||
"end_wall_ns": 1783870022707934728,
|
||||
"start_mono_ns": 201553631752443,
|
||||
"start_wall_ns": 1783869980758180054
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 586,
|
||||
"pass_rate": 0.03583617747440273,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "fe44bf6b0efd6e6ab4d19676e34ace608827e419dbb5c19c8595992f0339c9b1",
|
||||
"arrival_s": {
|
||||
"distinct_n": 585,
|
||||
"finite_n": 586,
|
||||
"max": 59.90990000000002,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"count": 586,
|
||||
"long_gt4096": 184,
|
||||
"offered_req_s": 9.766666666666667,
|
||||
"offered_req_s_per_gpu": 2.441666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 534,
|
||||
"finite_n": 586,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"raw_length_order_sha256": "e2d040068b279e99073f6965d1136eb0ca6e8534c883f331b4cb85c5fc7a4650",
|
||||
"request_id_order_sha256": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db"
|
||||
},
|
||||
"slo_pass_count": 21,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 199,
|
||||
"finite_n": 199,
|
||||
"max": 67.64735456709163,
|
||||
"min": 4.931260244077526,
|
||||
"missing_n": 387,
|
||||
"n": 586
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 199,
|
||||
"finite_n": 199,
|
||||
"max": 16151.948870014166,
|
||||
"min": 39.053958986187354,
|
||||
"missing_n": 387,
|
||||
"n": 586
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.034252608017, "kind": "anchor", "pass_rate": 0.8466666666666667, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.034252608017,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 581,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 63.947365009,
|
||||
"end_mono_ns": 201732794742162,
|
||||
"end_wall_ns": 1783870159921169450,
|
||||
"start_mono_ns": 201668847377153,
|
||||
"start_wall_ns": 1783870095973805093
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 600,
|
||||
"pass_rate": 0.8466666666666667,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "50f1b34c344c451c6d7501290d0b53f147d0c0b6a4bc078b303d5c33576ebe2d",
|
||||
"arrival_s": {
|
||||
"distinct_n": 599,
|
||||
"finite_n": 600,
|
||||
"max": 59.90990000000002,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 600
|
||||
},
|
||||
"count": 600,
|
||||
"long_gt4096": 187,
|
||||
"offered_req_s": 10.0,
|
||||
"offered_req_s_per_gpu": 2.5,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 546,
|
||||
"finite_n": 600,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 600
|
||||
},
|
||||
"raw_length_order_sha256": "08eff8eecd74dc4646aa45945bd752432ee5eb795006755527dd32d7178c1140",
|
||||
"request_id_order_sha256": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd"
|
||||
},
|
||||
"slo_pass_count": 508,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 581,
|
||||
"finite_n": 581,
|
||||
"max": 17.87731141735631,
|
||||
"min": 4.250030724219904,
|
||||
"missing_n": 19,
|
||||
"n": 600
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 581,
|
||||
"finite_n": 581,
|
||||
"max": 4941.07082701521,
|
||||
"min": 36.84570998302661,
|
||||
"missing_n": 19,
|
||||
"n": 600
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp4_mns16/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp4_mns16/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp4_mns16",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 26137,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/opprof/opprof-v1-dp0-pid2670831-1783869959685052499.jsonl"
|
||||
}
|
||||
6
runs/opprof-phase6/phase6/cells/tp4_mns16/commands.log
Normal file
6
runs/opprof-phase6/phase6/cells/tp4_mns16/commands.log
Normal file
@@ -0,0 +1,6 @@
|
||||
SERVER taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8521 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 4 --shutdown-timeout 120
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.033717411016 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/warmup
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.033717411016 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/anchor-0.033717411016
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.033182214016 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/anchor-0.033182214016
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.034252608017 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/anchor-0.034252608017
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.033717411016 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/confirm-1-anchor-0.033717411016
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.033717411016, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.033717411016,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 586,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.364052595,
|
||||
"end_mono_ns": 201800328727441,
|
||||
"end_wall_ns": 1783870227455155582,
|
||||
"start_mono_ns": 201738964674846,
|
||||
"start_wall_ns": 1783870166091102450
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 586,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "fe44bf6b0efd6e6ab4d19676e34ace608827e419dbb5c19c8595992f0339c9b1",
|
||||
"arrival_s": {
|
||||
"distinct_n": 585,
|
||||
"finite_n": 586,
|
||||
"max": 59.90990000000002,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"count": 586,
|
||||
"long_gt4096": 184,
|
||||
"offered_req_s": 9.766666666666667,
|
||||
"offered_req_s_per_gpu": 2.441666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 534,
|
||||
"finite_n": 586,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"raw_length_order_sha256": "e2d040068b279e99073f6965d1136eb0ca6e8534c883f331b4cb85c5fc7a4650",
|
||||
"request_id_order_sha256": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db"
|
||||
},
|
||||
"slo_pass_count": 586,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 586,
|
||||
"finite_n": 586,
|
||||
"max": 14.420212055125647,
|
||||
"min": 4.217363204660378,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 586,
|
||||
"finite_n": 586,
|
||||
"max": 819.4046519929543,
|
||||
"min": 36.395939998328686,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2670831-1783869959685052499.jsonl","encoded_records":26137,"written_records":26137,"dropped_records":0,"last_step_index":26136,"checkpoint_wall_ns":1783870239366621112,"flush_interval_seconds":1.0,"final":true}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user