gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Browse Source 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-07-13 11:06:10 +08:00
parent 607e88da3c
commit d5b276180d
412 changed files with 125056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

487
patches/vllm-0.24.0-opprof/0001-Add-lightweight-per-step-OpProf-telemetry.patch Normal file
View File

@@ -0,0 +1,487 @@
From f6f1cacbce0e39992d04843f652c1adda373ae43 Mon Sep 17 00:00:00 2001
From: Gahow Wang <gahow.wang@gmail.com>
Date: Sat, 11 Jul 2026 17:29:02 +0800
Subject: [PATCH 1/5] Add lightweight per-step OpProf telemetry
Assisted-by: OpenAI Codex
---
vllm/envs.py | 4 +
vllm/v1/core/sched/scheduler.py | 28 +++
vllm/v1/opprof.py | 337 +++++++++++++++++++++++++++++
vllm/v1/worker/gpu_model_runner.py | 6 +-
4 files changed, 374 insertions(+), 1 deletion(-)
create mode 100644 vllm/v1/opprof.py
diff --git a/vllm/envs.py b/vllm/envs.py
index 27a85bb..b3093e9 100755
--- a/vllm/envs.py
+++ b/vllm/envs.py
@@ -45,6 +45,7 @@ if TYPE_CHECKING:
VLLM_LOGGING_COLOR: str = "auto"
NO_COLOR: bool = False
VLLM_LOG_STATS_INTERVAL: float = 10.0
+ VLLM_OPPROF_DIR: str = ""
VLLM_TRACE_FUNCTION: int = 0
VLLM_USE_FLASHINFER_SAMPLER: bool = True
VLLM_PP_LAYER_PARTITION: str | None = None
@@ -786,6 +787,9 @@ environment_variables: dict[str, Callable[[], Any]] = {
if (val := float(os.getenv("VLLM_LOG_STATS_INTERVAL", "10."))) > 0.0
else 10.0
),
+ # Directory for per-step OpProf JSONL telemetry.
+ # Empty disables OpProf.
+ "VLLM_OPPROF_DIR": lambda: os.getenv("VLLM_OPPROF_DIR", ""),
# Trace function calls
# If set to 1, vllm will trace function calls
# Useful for debugging
diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py
index 90d93a1..303c562 100644
--- a/vllm/v1/core/sched/scheduler.py
+++ b/vllm/v1/core/sched/scheduler.py
@@ -7,6 +7,7 @@ from collections.abc import Iterable
from dataclasses import replace
from typing import Any
+import vllm.envs as envs
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import (
@@ -55,6 +56,7 @@ from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutp
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
from vllm.v1.metrics.stats import PrefixCacheStats, SchedulerStats
+from vllm.v1.opprof import OpProfRecorder
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
@@ -271,6 +273,12 @@ class Scheduler(SchedulerInterface):
if self.connector is not None:
self.connector.bind_gpu_block_pool(self.kv_cache_manager.block_pool)
+ self.opprof = OpProfRecorder.create(
+ output_dir=envs.VLLM_OPPROF_DIR,
+ dp_rank=self.parallel_config.data_parallel_index,
+ log_stats=self.log_stats,
+ )
+
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
self.use_v2_model_runner = vllm_config.use_v2_model_runner
# Scheduler iteration counter. Drives the V2+PP+async decode-throttle
@@ -386,6 +394,9 @@ class Scheduler(SchedulerInterface):
return num_new_tokens
def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput:
+ opprof_start = (
+ self.opprof.capture_start(self) if self.opprof is not None else None
+ )
self.current_step += 1
# NOTE(woosuk) on the scheduling algorithm:
# There's no "decoding phase" nor "prefill phase" in the scheduler.
@@ -1090,6 +1101,14 @@ class Scheduler(SchedulerInterface):
)
scheduler_output.ec_connector_metadata = ec_meta
+ if self.opprof is not None:
+ assert opprof_start is not None
+ self.opprof.begin(
+ scheduler=self,
+ output=scheduler_output,
+ start=opprof_start,
+ )
+
# Advance the fence only for non-empty steps (those that actually
# write KV and have their output processed later in update_from_output).
if self.defer_block_free and total_num_scheduled_tokens > 0:
@@ -1800,6 +1819,12 @@ class Scheduler(SchedulerInterface):
engine_core_outputs[0] = eco = EngineCoreOutputs()
eco.scheduler_stats = stats
+ if self.opprof is not None:
+ self.opprof.finalize(
+ output=scheduler_output,
+ cudagraph_stat=cudagraph_stats,
+ )
+
return engine_core_outputs
@staticmethod
@@ -2292,6 +2317,9 @@ class Scheduler(SchedulerInterface):
if self.ec_connector is not None:
self.ec_connector.shutdown()
+ if self.opprof is not None:
+ self.opprof.close()
+
logger.debug_once("[shutdown] Scheduler: complete")
########################################################################
diff --git a/vllm/v1/opprof.py b/vllm/v1/opprof.py
new file mode 100644
index 0000000..f0330d0
--- /dev/null
+++ b/vllm/v1/opprof.py
@@ -0,0 +1,337 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import atexit
+import logging
+import os
+import queue
+import threading
+import time
+from bisect import bisect_left
+from pathlib import Path
+from typing import Any
+
+import msgspec
+
+logger = logging.getLogger(__name__)
+
+SCHEMA_VERSION = 1
+CONTEXT_LENGTH_EDGES = tuple(1 << exponent for exponent in range(7, 18))
+CHUNK_SIZE_EDGES = tuple(1 << exponent for exponent in range(4, 12))
+DEFAULT_QUEUE_CAPACITY = 8192
+_CLOSE_TIMEOUT_SECONDS = 1.0
+_STOP = object()
+_PREFIX_FIELDS = ( # noqa: SIM905
+ "requests queries hits preempted_requests preempted_queries preempted_hits"
+).split()
+
+
+ScheduleStart = tuple[int, int, dict[str, dict[str, int] | None]]
+
+
+def classify_chunk(was_chunk: bool, end: int, target: int) -> str:
+ assert 0 <= end <= target
+ if was_chunk:
+ return "middle" if end < target else "final"
+ return "first" if end < target else "unsplit"
+
+
+def _prefix_values(stats: Any | None) -> dict[str, int] | None:
+ if stats is None:
+ return None
+ return {name: int(getattr(stats, name, 0)) for name in _PREFIX_FIELDS}
+
+
+def _prefix_snapshot(scheduler: Any) -> dict[str, dict[str, int] | None]:
+ return {
+ "local": _prefix_values(scheduler.kv_cache_manager.prefix_cache_stats),
+ "external": _prefix_values(scheduler.connector_prefix_cache_stats),
+ }
+
+
+def _prefix_delta(
+ before: dict[str, int] | None, after: dict[str, int] | None
+) -> dict[str, int] | None:
+ if before is None and after is None:
+ return None
+ before = before or dict.fromkeys(_PREFIX_FIELDS, 0)
+ after = after or dict.fromkeys(_PREFIX_FIELDS, 0)
+ delta = {name: after[name] - before[name] for name in _PREFIX_FIELDS}
+ assert all(value >= 0 for value in delta.values())
+ return delta
+
+
+class JSONLWriter:
+ def __init__(
+ self,
+ path: Path,
+ capacity: int = DEFAULT_QUEUE_CAPACITY,
+ start: bool = True,
+ ) -> None:
+ self._queue: queue.Queue[Any] = queue.Queue(capacity)
+ self._encoder = msgspec.json.Encoder()
+ self._file = path.open("xb", buffering=1 << 20)
+ self._thread = threading.Thread(target=self._run, daemon=True)
+ self._failure_lock = threading.Lock()
+ self._started = self._closed = False
+ self.failed = False
+ self.failure: Exception | None = None
+ self.encoded_records = self.written_records = 0
+ self.dropped_records = self._unreported_drops = 0
+ if start:
+ self.start()
+
+ def start(self) -> None:
+ if not self._started:
+ self._started = True
+ self._thread.start()
+
+ def _record_failure(self, error: Exception) -> None:
+ with self._failure_lock:
+ if self.failed:
+ return
+ self.failed = True
+ self.failure = error
+ logger.error("OpProf writer failed: %s", error)
+
+ def _writer_unavailable(self) -> bool:
+ if self.failed:
+ return True
+ if self._started and not self._thread.is_alive():
+ self._record_failure(RuntimeError("writer thread stopped unexpectedly"))
+ return True
+ return False
+
+ def _drop(self, pending: int) -> bool:
+ self.dropped_records += 1
+ self._unreported_drops = pending + 1
+ return False
+
+ def submit(self, record: dict[str, Any]) -> bool:
+ if self._closed:
+ raise RuntimeError("OpProf writer is closed")
+ pending = self._unreported_drops
+ record["dropped_records_before"] = pending
+ if self._writer_unavailable():
+ return self._drop(pending)
+ payload = self._encoder.encode(record) + b"\n"
+ self.encoded_records += 1
+ if self._writer_unavailable():
+ return self._drop(pending)
+ try:
+ self._queue.put_nowait(payload)
+ except queue.Full:
+ return self._drop(pending)
+ self._unreported_drops = 0
+ return True
+
+ def _run(self) -> None:
+ buffered = 0
+ last_flush = time.monotonic()
+ try:
+ while True:
+ try:
+ item = self._queue.get(timeout=1.0)
+ except queue.Empty:
+ self._file.flush()
+ buffered = 0
+ last_flush = time.monotonic()
+ continue
+ try:
+ if item is _STOP:
+ break
+ self._file.write(item)
+ buffered += len(item)
+ self.written_records += 1
+ now = time.monotonic()
+ if buffered >= 1 << 20 or now - last_flush >= 1.0:
+ self._file.flush()
+ buffered = 0
+ last_flush = now
+ finally:
+ self._queue.task_done()
+ except Exception as error:
+ self._record_failure(error)
+ finally:
+ footer = dict(
+ schema=SCHEMA_VERSION,
+ record_type="footer",
+ encoded_records=self.encoded_records,
+ written_records=self.written_records,
+ dropped_records=self.dropped_records,
+ )
+ try:
+ self._file.write(self._encoder.encode(footer) + b"\n")
+ self._file.flush()
+ except Exception as error:
+ self._record_failure(error)
+ finally:
+ try:
+ self._file.close()
+ except Exception as error:
+ self._record_failure(error)
+
+ def close(self) -> None:
+ if self._closed:
+ return
+ self._closed = True
+ self.start()
+ if not self._thread.is_alive():
+ return
+ try:
+ self._queue.put(_STOP, timeout=_CLOSE_TIMEOUT_SECONDS)
+ except queue.Full:
+ if not self._thread.is_alive():
+ return
+ try:
+ self._queue.put_nowait(_STOP)
+ except queue.Full:
+ self._record_failure(
+ TimeoutError("timed out enqueueing writer stop sentinel")
+ )
+ return
+ self._thread.join(timeout=_CLOSE_TIMEOUT_SECONDS)
+ if self._thread.is_alive():
+ self._record_failure(TimeoutError("timed out joining writer thread"))
+
+
+class OpProfRecorder:
+ def __init__(self, engine_id: str, writer: JSONLWriter) -> None:
+ self.engine_id = engine_id
+ self.writer = writer
+ self._next_step = 0
+ self._pending: dict[int, dict[str, Any]] = {}
+ atexit.register(self.close)
+
+ @classmethod
+ def create(
+ cls, output_dir: str, dp_rank: int, log_stats: bool
+ ) -> "OpProfRecorder | None":
+ if not output_dir:
+ return None
+ if not log_stats:
+ raise ValueError("VLLM_OPPROF_DIR requires log stats to be enabled")
+ directory = Path(output_dir).expanduser()
+ if not directory.is_absolute():
+ raise ValueError("VLLM_OPPROF_DIR must be an absolute path")
+ directory.mkdir(parents=True, exist_ok=True)
+ engine_id = f"dp{dp_rank}-pid{os.getpid()}"
+ name = f"opprof-v{SCHEMA_VERSION}-{engine_id}-{time.time_ns()}.jsonl"
+ return cls(engine_id, JSONLWriter(directory / name))
+
+ @staticmethod
+ def capture_start(scheduler: Any) -> ScheduleStart:
+ return time.time_ns(), time.monotonic_ns(), _prefix_snapshot(scheduler)
+
+ def begin(self, scheduler: Any, output: Any, start: ScheduleStart) -> None:
+ key = id(output)
+ assert key not in self._pending, "duplicate OpProf begin"
+ new_ids = {request.req_id for request in output.scheduled_new_reqs}
+ prefill_requests = prefill_tokens = decode_requests = decode_tokens = 0
+ context_length_hist = [0] * (len(CONTEXT_LENGTH_EDGES) + 1)
+ chunk_size_hist = [0] * (len(CHUNK_SIZE_EDGES) + 1)
+ chunks: dict[str, Any] = dict.fromkeys(
+ ("first", "middle", "final", "unsplit"), 0
+ )
+ for req_id, num_tokens in output.num_scheduled_tokens.items():
+ request = scheduler.requests[req_id]
+ end = request.num_computed_tokens + num_tokens
+ assert end >= 0
+ context_length_hist[bisect_left(CONTEXT_LENGTH_EDGES, end)] += 1
+ is_prefill = (
+ req_id in new_ids
+ or output.scheduled_cached_reqs.is_context_phase(req_id)
+ )
+ if is_prefill:
+ prefill_requests += 1
+ prefill_tokens += num_tokens
+ assert num_tokens >= 0
+ chunk_size_hist[bisect_left(CHUNK_SIZE_EDGES, num_tokens)] += 1
+ target = request.num_tokens + request.num_output_placeholders
+ chunks[classify_chunk(request.is_prefill_chunk, end, target)] += 1
+ else:
+ decode_requests += 1
+ decode_tokens += num_tokens
+ prefix_after = _prefix_snapshot(scheduler)
+ block_pool = scheduler.kv_cache_manager.block_pool
+ total_blocks = block_pool.num_gpu_blocks - 1
+ free_blocks = block_pool.get_num_free_blocks()
+ assert 0 <= free_blocks <= total_blocks
+ chunks["tokens"] = prefill_tokens
+ chunks["chunk_size_hist"] = chunk_size_hist
+ values = dict(
+ schema=SCHEMA_VERSION,
+ engine_id=self.engine_id,
+ step_index=self._next_step,
+ submit_wall_ns=start[0],
+ submit_mono_ns=start[1],
+ model_executed=output.total_num_scheduled_tokens > 0,
+ scheduled_requests=len(output.num_scheduled_tokens),
+ decode_batch_size=decode_requests,
+ prefill_requests=prefill_requests,
+ prefill_tokens=prefill_tokens,
+ decode_tokens=decode_tokens,
+ chunked_prefill=chunks,
+ context_length_hist=context_length_hist,
+ preemptions=len(output.preempted_req_ids or ()),
+ queues=dict(
+ running=len(scheduler.running),
+ waiting=len(scheduler.waiting),
+ deferred=len(scheduler.skipped_waiting),
+ ),
+ kv=dict(
+ total_blocks=total_blocks,
+ free_blocks=free_blocks,
+ used_blocks=total_blocks - free_blocks,
+ usage=scheduler.kv_cache_manager.usage,
+ ),
+ prefix=dict(
+ local=_prefix_delta(start[2]["local"], prefix_after["local"]),
+ external=_prefix_delta(start[2]["external"], prefix_after["external"]),
+ ),
+ )
+ assert prefill_tokens + decode_tokens == output.total_num_scheduled_tokens
+ self._pending[key] = values
+ self._next_step += 1
+
+ def finalize(self, output: Any, cudagraph_stat: Any | None) -> bool:
+ try:
+ values = self._pending.pop(id(output))
+ except KeyError:
+ raise AssertionError("missing or already finalized OpProf step") from None
+ if cudagraph_stat is None:
+ assert not values["model_executed"]
+ cudagraph = dict(
+ hit=False,
+ runtime_mode="NONE",
+ unpadded_tokens=0,
+ bucket_tokens=0,
+ padding_tokens=0,
+ )
+ else:
+ mode = str(cudagraph_stat.runtime_mode).rsplit(".", 1)[-1]
+ cudagraph = dict(
+ hit=mode != "NONE",
+ runtime_mode=mode,
+ unpadded_tokens=cudagraph_stat.num_unpadded_tokens,
+ bucket_tokens=cudagraph_stat.num_padded_tokens,
+ padding_tokens=cudagraph_stat.num_paddings,
+ )
+ record = dict(
+ values,
+ complete_mono_ns=time.monotonic_ns(),
+ cudagraph=cudagraph,
+ moe_expert_load=None,
+ dropped_records_before=0,
+ )
+ return self.writer.submit(record)
+
+ def close(self) -> None:
+ self.writer.close()
+
+ @property
+ def failed(self) -> bool:
+ return self.writer.failed
+
+ @property
+ def failure(self) -> Exception | None:
+ return self.writer.failure
diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py
index 74938a8..c11d773 100644
--- a/vllm/v1/worker/gpu_model_runner.py
+++ b/vllm/v1/worker/gpu_model_runner.py
@@ -437,6 +437,7 @@ class GPUModelRunner(
self.scheduler_config = vllm_config.scheduler_config
self.speculative_config = vllm_config.speculative_config
self.observability_config = vllm_config.observability_config
+ self.opprof_enabled = bool(envs.VLLM_OPPROF_DIR)
model_config = self.model_config
cache_config = self.cache_config
@@ -3917,7 +3918,10 @@ class GPUModelRunner(
assert batch_descriptor.num_tokens == num_tokens_padded
cudagraph_stats = None
- if self.vllm_config.observability_config.cudagraph_metrics:
+ if (
+ self.vllm_config.observability_config.cudagraph_metrics
+ or self.opprof_enabled
+ ):
cudagraph_stats = CUDAGraphStat(
num_unpadded_tokens=num_tokens,
num_padded_tokens=batch_descriptor.num_tokens,
--
2.43.0

417
patches/vllm-0.24.0-opprof/0002-Add-standalone-OpProf-telemetry-tests.patch Normal file
View File

@@ -0,0 +1,417 @@
From 4f4ee674f217698436b00c3ab6357f59a792477a Mon Sep 17 00:00:00 2001
From: Gahow Wang <gahow.wang@gmail.com>
Date: Sat, 11 Jul 2026 17:29:02 +0800
Subject: [PATCH 2/5] Add standalone OpProf telemetry tests
Assisted-by: OpenAI Codex
---
tests/v1/core/test_opprof.py | 397 +++++++++++++++++++++++++++++++++++
1 file changed, 397 insertions(+)
create mode 100644 tests/v1/core/test_opprof.py
diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
new file mode 100644
index 0000000..9bfbfcc
--- /dev/null
+++ b/tests/v1/core/test_opprof.py
@@ -0,0 +1,397 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Standalone tests: this file intentionally does not import the vllm package."""
+
+import errno
+import importlib.util
+import logging
+import sys
+import threading
+from pathlib import Path
+from types import SimpleNamespace
+
+import msgspec
+import pytest
+
+_ROOT = Path(__file__).parents[3]
+_SPEC = importlib.util.spec_from_file_location(
+ "opprof_standalone", _ROOT / "vllm" / "v1" / "opprof.py"
+)
+assert _SPEC is not None and _SPEC.loader is not None
+opprof = importlib.util.module_from_spec(_SPEC)
+sys.modules[_SPEC.name] = opprof
+_SPEC.loader.exec_module(opprof)
+
+
+class CachedRequests:
+ def __init__(self, context_ids=()):
+ self.context_ids = set(context_ids)
+
+ def is_context_phase(self, req_id):
+ return req_id in self.context_ids
+
+
+def prefix_stats(**overrides):
+ values = dict.fromkeys(opprof._PREFIX_FIELDS, 0)
+ values.update(overrides)
+ return SimpleNamespace(**values)
+
+
+def request(computed, total, was_chunk=False, placeholders=0):
+ return SimpleNamespace(
+ num_computed_tokens=computed,
+ num_tokens=total,
+ num_output_placeholders=placeholders,
+ is_prefill_chunk=was_chunk,
+ )
+
+
+def scheduler(requests, local=None, external=None):
+ block_pool = SimpleNamespace(
+ num_gpu_blocks=101,
+ get_num_free_blocks=lambda: 40,
+ )
+ kv_manager = SimpleNamespace(
+ prefix_cache_stats=local or prefix_stats(),
+ block_pool=block_pool,
+ usage=0.6,
+ )
+ return SimpleNamespace(
+ requests=requests,
+ kv_cache_manager=kv_manager,
+ connector_prefix_cache_stats=external,
+ running=list(range(len(requests))),
+ waiting=[0, 1],
+ skipped_waiting=[0],
+ )
+
+
+def schedule_output(tokens, context_ids=(), new_ids=(), preempted=()):
+ return SimpleNamespace(
+ scheduled_new_reqs=[SimpleNamespace(req_id=req_id) for req_id in new_ids],
+ scheduled_cached_reqs=CachedRequests(context_ids),
+ num_scheduled_tokens=tokens,
+ total_num_scheduled_tokens=sum(tokens.values()),
+ preempted_req_ids=set(preempted),
+ )
+
+
+def graph(mode="FULL", unpadded=1, padded=1):
+ return SimpleNamespace(
+ runtime_mode=mode,
+ num_unpadded_tokens=unpadded,
+ num_padded_tokens=padded,
+ num_paddings=padded - unpadded,
+ )
+
+
+def recorder(tmp_path, *, capacity=8192, start=True):
+ path = tmp_path / "opprof.jsonl"
+ writer = opprof.JSONLWriter(path, capacity=capacity, start=start)
+ return opprof.OpProfRecorder("dp0-pid1", writer), path
+
+
+def emit(rec, sched, output, cg=None):
+ start = rec.capture_start(sched)
+ rec.begin(sched, output, start)
+ return rec.finalize(output, cg or graph())
+
+
+def read_jsonl(path):
+ return [msgspec.json.decode(line) for line in path.read_bytes().splitlines()]
+
+
+def test_import_light_and_approved_constants():
+ assert "torch" not in sys.modules
+ assert "vllm" not in sys.modules
+ assert opprof.DEFAULT_QUEUE_CAPACITY == 8192
+ assert tuple(1 << i for i in range(7, 18)) == opprof.CONTEXT_LENGTH_EDGES
+ assert tuple(1 << i for i in range(4, 12)) == opprof.CHUNK_SIZE_EDGES
+
+
+def test_schema_and_invariants(tmp_path):
+ sched = scheduler(
+ {
+ "first": request(0, 100),
+ "final": request(64, 100, was_chunk=True),
+ "decode": request(1024, 1025),
+ }
+ )
+ output = schedule_output(
+ {"first": 64, "final": 36, "decode": 1},
+ context_ids={"final"},
+ new_ids={"first"},
+ preempted={"old"},
+ )
+ rec, path = recorder(tmp_path)
+ start = rec.capture_start(sched)
+ sched.kv_cache_manager.prefix_cache_stats = prefix_stats(
+ requests=1, queries=100, hits=64
+ )
+ rec.begin(sched, output, start)
+ assert rec.finalize(output, graph("FULL", 101, 128))
+ rec.close()
+
+ record, footer = read_jsonl(path)
+ assert record["schema"] == 1
+ assert record["scheduled_requests"] == 3
+ assert record["prefill_requests"] == 2
+ assert record["decode_batch_size"] == 1
+ assert record["prefill_tokens"] + record["decode_tokens"] == 101
+ assert sum(record["context_length_hist"]) == 3
+ assert len(record["context_length_hist"]) == 12
+ assert sum(record["chunked_prefill"]["chunk_size_hist"]) == 2
+ assert len(record["chunked_prefill"]["chunk_size_hist"]) == 9
+ assert record["chunked_prefill"]["first"] == 1
+ assert record["chunked_prefill"]["final"] == 1
+ assert record["preemptions"] == 1
+ assert record["kv"] == {
+ "total_blocks": 100,
+ "free_blocks": 40,
+ "used_blocks": 60,
+ "usage": 0.6,
+ }
+ assert record["prefix"]["local"]["hits"] == 64
+ assert record["moe_expert_load"] is None
+ assert record["complete_mono_ns"] >= record["submit_mono_ns"]
+ assert footer["record_type"] == "footer"
+ assert footer["written_records"] == 1
+
+
+def test_capture_record_matches_pre_refactor_golden(tmp_path, monkeypatch):
+ sched = scheduler(
+ {
+ "edge": request(0, 128),
+ "after": request(65, 129, was_chunk=True),
+ "decode": request(256, 257),
+ }
+ )
+ output = schedule_output(
+ {"edge": 128, "after": 64, "decode": 1},
+ context_ids={"after"},
+ new_ids={"edge"},
+ preempted={"old"},
+ )
+ rec, path = recorder(tmp_path)
+ zero_prefix = dict.fromkeys(opprof._PREFIX_FIELDS, 0)
+ start = (100, 200, {"local": zero_prefix, "external": None})
+ monkeypatch.setattr(opprof.time, "monotonic_ns", lambda: 300)
+
+ rec.begin(sched, output, start)
+ assert rec.finalize(output, graph("FULL", 193, 256))
+ rec.close()
+
+ record = read_jsonl(path)[0]
+ assert record == {
+ "schema": 1,
+ "engine_id": "dp0-pid1",
+ "step_index": 0,
+ "submit_wall_ns": 100,
+ "submit_mono_ns": 200,
+ "model_executed": True,
+ "scheduled_requests": 3,
+ "decode_batch_size": 1,
+ "prefill_requests": 2,
+ "prefill_tokens": 192,
+ "decode_tokens": 1,
+ "chunked_prefill": {
+ "first": 0,
+ "middle": 0,
+ "final": 1,
+ "unsplit": 1,
+ "tokens": 192,
+ "chunk_size_hist": [0, 0, 1, 1, 0, 0, 0, 0, 0],
+ },
+ "context_length_hist": [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+ "preemptions": 1,
+ "queues": {"running": 3, "waiting": 2, "deferred": 1},
+ "kv": {
+ "total_blocks": 100,
+ "free_blocks": 40,
+ "used_blocks": 60,
+ "usage": 0.6,
+ },
+ "prefix": {"local": zero_prefix, "external": None},
+ "complete_mono_ns": 300,
+ "cudagraph": {
+ "hit": True,
+ "runtime_mode": "FULL",
+ "unpadded_tokens": 193,
+ "bucket_tokens": 256,
+ "padding_tokens": 63,
+ },
+ "moe_expert_load": None,
+ "dropped_records_before": 0,
+ }
+
+
+@pytest.mark.parametrize(
+ ("was_chunk", "end", "target", "expected"),
+ [
+ (False, 64, 100, "first"),
+ (True, 80, 100, "middle"),
+ (True, 100, 100, "final"),
+ (False, 100, 100, "unsplit"),
+ ],
+)
+def test_chunk_classification(was_chunk, end, target, expected):
+ assert opprof.classify_chunk(was_chunk, end, target) == expected
+
+
+def test_async_pairing_out_of_order_and_double_finalize(tmp_path):
+ sched = scheduler({"a": request(10, 11), "b": request(200, 201)})
+ first = schedule_output({"a": 1})
+ second = schedule_output({"b": 1})
+ rec, path = recorder(tmp_path)
+ rec.begin(sched, first, rec.capture_start(sched))
+ rec.begin(sched, second, rec.capture_start(sched))
+ assert rec.finalize(second, graph())
+ assert rec.finalize(first, graph("NONE"))
+ with pytest.raises(AssertionError, match="already finalized"):
+ rec.finalize(second, graph())
+ assert not rec._pending
+ rec.close()
+ records = read_jsonl(path)[:-1]
+ assert [record["step_index"] for record in records] == [1, 0]
+ assert records[0]["context_length_hist"][1] == 1
+ assert records[1]["context_length_hist"][0] == 1
+
+
+def test_disabled_noop_and_log_stats_fail_fast(tmp_path):
+ assert opprof.OpProfRecorder.create("", dp_rank=0, log_stats=False) is None
+ with pytest.raises(ValueError, match="requires log stats"):
+ opprof.OpProfRecorder.create(str(tmp_path), dp_rank=0, log_stats=False)
+ assert not list(tmp_path.iterdir())
+
+
+def test_bounded_queue_drop_accounting(tmp_path):
+ sched = scheduler({str(i): request(i, i + 1) for i in range(3)})
+ rec, path = recorder(tmp_path, capacity=1, start=False)
+ assert emit(rec, sched, schedule_output({"0": 1}))
+ assert not emit(rec, sched, schedule_output({"1": 1}))
+ rec.writer.start()
+ rec.writer._queue.join()
+ assert emit(rec, sched, schedule_output({"2": 1}))
+ rec.close()
+
+ first, after_drop, footer = read_jsonl(path)
+ assert first["step_index"] == 0
+ assert after_drop["step_index"] == 2
+ assert after_drop["dropped_records_before"] == 1
+ assert footer["encoded_records"] == 3
+ assert footer["written_records"] == 2
+ assert footer["dropped_records"] == 1
+
+
+def test_writer_enospc_is_exposed_and_shutdown_is_bounded(
+ tmp_path, monkeypatch, caplog
+):
+ sched = scheduler(
+ {
+ "first": request(0, 1),
+ "after_failure": request(1, 2),
+ }
+ )
+ rec, _ = recorder(tmp_path, capacity=1, start=False)
+ assert emit(rec, sched, schedule_output({"first": 1}))
+
+ real_file = rec.writer._file
+
+ def fail_enospc(*_args, **_kwargs):
+ raise OSError(errno.ENOSPC, "No space left on device")
+
+ failing_file = SimpleNamespace(
+ write=fail_enospc,
+ flush=fail_enospc,
+ close=real_file.close,
+ )
+ monkeypatch.setattr(rec.writer, "_file", failing_file)
+ caplog.set_level(logging.ERROR, logger=opprof.__name__)
+
+ rec.writer.start()
+ rec.writer._thread.join(timeout=1.0)
+ assert not rec.writer._thread.is_alive()
+
+ producer_result = emit(
+ rec, sched, schedule_output({"after_failure": 1})
+ )
+
+ closer = threading.Thread(target=rec.close, daemon=True)
+ closer.start()
+ closer.join(timeout=1.0)
+ assert not closer.is_alive(), "OpProf close blocked after writer failure"
+ assert not producer_result
+ assert rec.writer.dropped_records == 1
+ assert rec.failed
+ assert isinstance(rec.failure, OSError)
+ assert rec.failure.errno == errno.ENOSPC
+ errors = [
+ record
+ for record in caplog.records
+ if "OpProf writer failed" in record.getMessage()
+ ]
+ assert len(errors) == 1
+
+
+def test_shutdown_flush_is_idempotent(tmp_path):
+ sched = scheduler({"decode": request(8, 9)})
+ rec, path = recorder(tmp_path)
+ assert emit(rec, sched, schedule_output({"decode": 1}))
+ rec.close()
+ rec.close()
+ record, footer = read_jsonl(path)
+ assert record["step_index"] == 0
+ assert footer["written_records"] == 1
+ assert path.stat().st_size > 0
+
+
+def test_piecewise_cudagraph_record_preserved(tmp_path):
+ sched = scheduler({"decode": request(4096, 4097)})
+ output = schedule_output({"decode": 1})
+ rec, path = recorder(tmp_path)
+ assert emit(rec, sched, output, graph("PIECEWISE", 513, 520))
+ rec.close()
+ record = read_jsonl(path)[0]
+ assert record["cudagraph"] == {
+ "hit": True,
+ "runtime_mode": "PIECEWISE",
+ "unpadded_tokens": 513,
+ "bucket_tokens": 520,
+ "padding_tokens": 7,
+ }
+
+
+def test_zero_scheduled_tokens_finalize_without_cudagraph(tmp_path):
+ sched = scheduler({})
+ output = schedule_output({})
+ rec, path = recorder(tmp_path)
+ rec.begin(sched, output, rec.capture_start(sched))
+ assert rec._pending
+
+ assert rec.finalize(output, cudagraph_stat=None)
+ assert not rec._pending
+ rec.close()
+
+ record = read_jsonl(path)[0]
+ assert record["model_executed"] is False
+ assert record["scheduled_requests"] == 0
+ assert record["prefill_requests"] == 0
+ assert record["prefill_tokens"] == 0
+ assert record["decode_batch_size"] == 0
+ assert record["decode_tokens"] == 0
+ assert record["context_length_hist"] == [0] * 12
+ assert record["chunked_prefill"] == {
+ "first": 0,
+ "middle": 0,
+ "final": 0,
+ "unsplit": 0,
+ "tokens": 0,
+ "chunk_size_hist": [0] * 9,
+ }
+ assert record["cudagraph"] == {
+ "hit": False,
+ "runtime_mode": "NONE",
+ "unpadded_tokens": 0,
+ "bucket_tokens": 0,
+ "padding_tokens": 0,
+ }
--
2.43.0

52
patches/vllm-0.24.0-opprof/0003-Log-the-OpProf-output-path-at-startup.patch Normal file
View File

@@ -0,0 +1,52 @@
From 668cfb7e27e488454dbf09a4927b8a60d6d49b40 Mon Sep 17 00:00:00 2001
From: Gahow Wang <gahow.wang@gmail.com>
Date: Sat, 11 Jul 2026 17:32:27 +0800
Subject: [PATCH 3/5] Log the OpProf output path at startup
Assisted-by: OpenAI Codex
---
tests/v1/core/test_opprof.py | 1 +
vllm/v1/core/sched/scheduler.py | 2 ++
vllm/v1/opprof.py | 1 +
3 files changed, 4 insertions(+)
diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
index 9bfbfcc..79c1fae 100644
--- a/tests/v1/core/test_opprof.py
+++ b/tests/v1/core/test_opprof.py
@@ -336,6 +336,7 @@ def test_writer_enospc_is_exposed_and_shutdown_is_bounded(
def test_shutdown_flush_is_idempotent(tmp_path):
sched = scheduler({"decode": request(8, 9)})
rec, path = recorder(tmp_path)
+ assert rec.writer.path == path
assert emit(rec, sched, schedule_output({"decode": 1}))
rec.close()
rec.close()
diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py
index 303c562..769a02a 100644
--- a/vllm/v1/core/sched/scheduler.py
+++ b/vllm/v1/core/sched/scheduler.py
@@ -278,6 +278,8 @@ class Scheduler(SchedulerInterface):
dp_rank=self.parallel_config.data_parallel_index,
log_stats=self.log_stats,
)
+ if self.opprof is not None:
+ logger.info("OpProf telemetry enabled: %s", self.opprof.writer.path)
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
self.use_v2_model_runner = vllm_config.use_v2_model_runner
diff --git a/vllm/v1/opprof.py b/vllm/v1/opprof.py
index f0330d0..75f63de 100644
--- a/vllm/v1/opprof.py
+++ b/vllm/v1/opprof.py
@@ -68,6 +68,7 @@ class JSONLWriter:
start: bool = True,
) -> None:
self._queue: queue.Queue[Any] = queue.Queue(capacity)
+ self.path = path
self._encoder = msgspec.json.Encoder()
self._file = path.open("xb", buffering=1 << 20)
self._thread = threading.Thread(target=self._run, daemon=True)
--
2.43.0

64
patches/vllm-0.24.0-opprof/0004-Exclude-OpProf-output-path-from-compile-cache-key.patch Normal file
View File

@@ -0,0 +1,64 @@
From 335da4abe60e0177872e0b7751e86eeec6756a2b Mon Sep 17 00:00:00 2001
From: Gahow Wang <gahow.wang@gmail.com>
Date: Sat, 11 Jul 2026 22:32:30 +0800
Subject: [PATCH 4/5] Exclude OpProf output path from compile cache key
---
tests/v1/core/test_opprof.py | 21 ++++++++++++++++++++-
vllm/envs.py | 1 +
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
index 79c1fae..a820e7e 100644
--- a/tests/v1/core/test_opprof.py
+++ b/tests/v1/core/test_opprof.py
@@ -8,7 +8,7 @@ import logging
import sys
import threading
from pathlib import Path
-from types import SimpleNamespace
+from types import ModuleType, SimpleNamespace
import msgspec
import pytest
@@ -23,6 +23,25 @@ sys.modules[_SPEC.name] = opprof
_SPEC.loader.exec_module(opprof)
+def test_output_dir_is_not_a_compile_factor(monkeypatch: pytest.MonkeyPatch):
+ spec = importlib.util.spec_from_file_location(
+ "envs_standalone", _ROOT / "vllm" / "envs.py"
+ )
+ assert spec is not None and spec.loader is not None
+ envs = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, spec.name, envs)
+ spec.loader.exec_module(envs)
+
+ monkeypatch.setitem(sys.modules, "vllm", ModuleType("vllm"))
+ monkeypatch.setitem(sys.modules, "vllm.config", ModuleType("vllm.config"))
+ config_utils = ModuleType("vllm.config.utils")
+ config_utils.__dict__["normalize_value"] = lambda value: value
+ monkeypatch.setitem(sys.modules, "vllm.config.utils", config_utils)
+ monkeypatch.setenv("VLLM_OPPROF_DIR", "/tmp/opprof")
+
+ assert "VLLM_OPPROF_DIR" not in envs.compile_factors()
+
+
class CachedRequests:
def __init__(self, context_ids=()):
self.context_ids = set(context_ids)
diff --git a/vllm/envs.py b/vllm/envs.py
index b3093e9..5634708 100755
--- a/vllm/envs.py
+++ b/vllm/envs.py
@@ -2044,6 +2044,7 @@ def compile_factors() -> dict[str, object]:
"VLLM_LOGGING_CONFIG_PATH",
"VLLM_LOGGING_COLOR",
"VLLM_LOG_STATS_INTERVAL",
+ "VLLM_OPPROF_DIR",
"VLLM_DEBUG_LOG_API_SERVER_RESPONSE",
"VLLM_TUNED_CONFIG_FOLDER",
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR",
--
2.43.0

24
patches/vllm-0.24.0-opprof/0005-Keep-compile-factor-regression-import-light.patch Normal file
View File

@@ -0,0 +1,24 @@
From bbfa7176a6a3686a88ee66696f1ad8d754559d96 Mon Sep 17 00:00:00 2001
From: Gahow Wang <gahow.wang@gmail.com>
Date: Sat, 11 Jul 2026 22:38:08 +0800
Subject: [PATCH 5/5] Keep compile-factor regression import-light
---
tests/v1/core/test_opprof.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
index a820e7e..0b8a8a1 100644
--- a/tests/v1/core/test_opprof.py
+++ b/tests/v1/core/test_opprof.py
@@ -37,6 +37,7 @@ def test_output_dir_is_not_a_compile_factor(monkeypatch: pytest.MonkeyPatch):
config_utils = ModuleType("vllm.config.utils")
config_utils.__dict__["normalize_value"] = lambda value: value
monkeypatch.setitem(sys.modules, "vllm.config.utils", config_utils)
+ monkeypatch.setitem(sys.modules, "torch", ModuleType("torch"))
monkeypatch.setenv("VLLM_OPPROF_DIR", "/tmp/opprof")
assert "VLLM_OPPROF_DIR" not in envs.compile_factors()
--
2.43.0

306
patches/vllm-0.24.0-opprof/0006-Checkpoint-OpProf-accounting-across-hard-kills.patch Normal file
View File

@@ -0,0 +1,306 @@
From f8b68f2452c424d22de4a69527a427207dcfbca5 Mon Sep 17 00:00:00 2001
From: Gahow Wang <gahow.wang@gmail.com>
Date: Sun, 12 Jul 2026 12:56:39 +0800
Subject: [PATCH] Checkpoint OpProf accounting across hard kills
---
tests/v1/core/test_opprof.py | 118 +++++++++++++++++++++++++++++++++++
vllm/v1/opprof.py | 84 ++++++++++++++++++++++---
2 files changed, 195 insertions(+), 7 deletions(-)
diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
index 0b8a8a1..007c5bb 100644
--- a/tests/v1/core/test_opprof.py
+++ b/tests/v1/core/test_opprof.py
@@ -5,6 +5,8 @@
import errno
import importlib.util
import logging
+import os
+import subprocess
import sys
import threading
from pathlib import Path
@@ -121,6 +123,12 @@ def read_jsonl(path):
return [msgspec.json.decode(line) for line in path.read_bytes().splitlines()]
+def read_sidecar(path):
+ return msgspec.json.decode(
+ path.with_name(f"{path.name}.footer.json").read_bytes()
+ )
+
+
def test_import_light_and_approved_constants():
assert "torch" not in sys.modules
assert "vllm" not in sys.modules
@@ -366,6 +374,116 @@ def test_shutdown_flush_is_idempotent(tmp_path):
assert path.stat().st_size > 0
+def test_sidecar_updates_are_atomic(tmp_path, monkeypatch):
+ writer = opprof.JSONLWriter(tmp_path / "atomic.jsonl", start=False)
+ writer._write_sidecar(
+ encoded_records=1,
+ written_records=1,
+ dropped_records=0,
+ last_step_index=0,
+ final=False,
+ )
+ original = writer.sidecar_path.read_bytes()
+ real_replace = os.replace
+ replacements = []
+
+ def inspect_replace(source, destination):
+ assert Path(destination) == writer.sidecar_path
+ assert writer.sidecar_path.read_bytes() == original
+ candidate = msgspec.json.decode(Path(source).read_bytes())
+ assert candidate["written_records"] == 2
+ replacements.append(candidate)
+ real_replace(source, destination)
+
+ monkeypatch.setattr(opprof.os, "replace", inspect_replace)
+ writer._write_sidecar(
+ encoded_records=3,
+ written_records=2,
+ dropped_records=1,
+ last_step_index=2,
+ final=False,
+ )
+ monkeypatch.setattr(opprof.os, "replace", real_replace)
+
+ assert len(replacements) == 1
+ assert read_sidecar(writer.path)["encoded_records"] == 3
+ assert not list(tmp_path.glob(".*.tmp-*"))
+ writer.close()
+
+
+def test_hard_kill_sidecar_balances_last_flush(tmp_path):
+ path = tmp_path / "hard-kill.jsonl"
+ child = """
+import importlib.util
+import sys
+import time
+from pathlib import Path
+
+source, output = sys.argv[1:]
+spec = importlib.util.spec_from_file_location("opprof_hard_kill", source)
+module = importlib.util.module_from_spec(spec)
+sys.modules[spec.name] = module
+spec.loader.exec_module(module)
+writer = module.JSONLWriter(Path(output))
+for step in range(3):
+ assert writer.submit({"schema": 1, "step_index": step})
+writer._queue.join()
+while not writer.sidecar_path.exists():
+ time.sleep(0.01)
+print("READY", flush=True)
+time.sleep(60)
+"""
+ process = subprocess.Popen(
+ [
+ sys.executable,
+ "-c",
+ child,
+ str(_ROOT / "vllm/v1/opprof.py"),
+ str(path),
+ ],
+ stdout=subprocess.PIPE,
+ text=True,
+ )
+ try:
+ assert process.stdout is not None
+ assert process.stdout.readline().strip() == "READY"
+ process.kill()
+ assert process.wait(timeout=5) < 0
+ finally:
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout=5)
+
+ records = read_jsonl(path)
+ sidecar = read_sidecar(path)
+ assert len(records) == 3
+ assert all(record.get("record_type") != "footer" for record in records)
+ assert sidecar["final"] is False
+ assert sidecar["written_records"] == len(records)
+ assert sidecar["encoded_records"] == (
+ sidecar["written_records"] + sidecar["dropped_records"]
+ )
+ assert sidecar["last_step_index"] == records[-1]["step_index"] == 2
+
+
+def test_clean_footer_and_final_sidecar_agree(tmp_path):
+ sched = scheduler({"decode": request(8, 9)})
+ rec, path = recorder(tmp_path)
+ assert emit(rec, sched, schedule_output({"decode": 1}))
+ rec.close()
+
+ record, footer = read_jsonl(path)
+ sidecar = read_sidecar(path)
+ assert sidecar["record_type"] == "footer_checkpoint"
+ assert sidecar["stream"] == path.name
+ assert sidecar["final"] is True
+ assert sidecar["last_step_index"] == record["step_index"] == 0
+ assert sidecar["checkpoint_wall_ns"] > 0
+ assert sidecar["flush_interval_seconds"] == opprof.FLUSH_INTERVAL_SECONDS
+ for counter in ("encoded_records", "written_records", "dropped_records"):
+ assert sidecar[counter] == footer[counter]
+
+
def test_piecewise_cudagraph_record_preserved(tmp_path):
sched = scheduler({"decode": request(4096, 4097)})
output = schedule_output({"decode": 1})
diff --git a/vllm/v1/opprof.py b/vllm/v1/opprof.py
index 75f63de..28d9635 100644
--- a/vllm/v1/opprof.py
+++ b/vllm/v1/opprof.py
@@ -7,6 +7,7 @@ import queue
import threading
import time
from bisect import bisect_left
+from contextlib import suppress
from pathlib import Path
from typing import Any
@@ -18,6 +19,7 @@ SCHEMA_VERSION = 1
CONTEXT_LENGTH_EDGES = tuple(1 << exponent for exponent in range(7, 18))
CHUNK_SIZE_EDGES = tuple(1 << exponent for exponent in range(4, 12))
DEFAULT_QUEUE_CAPACITY = 8192
+FLUSH_INTERVAL_SECONDS = 1.0
_CLOSE_TIMEOUT_SECONDS = 1.0
_STOP = object()
_PREFIX_FIELDS = ( # noqa: SIM905
@@ -69,6 +71,7 @@ class JSONLWriter:
) -> None:
self._queue: queue.Queue[Any] = queue.Queue(capacity)
self.path = path
+ self.sidecar_path = path.with_name(f"{path.name}.footer.json")
self._encoder = msgspec.json.Encoder()
self._file = path.open("xb", buffering=1 << 20)
self._thread = threading.Thread(target=self._run, daemon=True)
@@ -78,6 +81,9 @@ class JSONLWriter:
self.failure: Exception | None = None
self.encoded_records = self.written_records = 0
self.dropped_records = self._unreported_drops = 0
+ self._checkpoint_encoded_records = 0
+ self._checkpoint_dropped_records = 0
+ self._last_written_step_index: int | None = None
if start:
self.start()
@@ -119,33 +125,90 @@ class JSONLWriter:
if self._writer_unavailable():
return self._drop(pending)
try:
- self._queue.put_nowait(payload)
+ self._queue.put_nowait(
+ (
+ payload,
+ self.encoded_records,
+ self.dropped_records,
+ int(record["step_index"]),
+ )
+ )
except queue.Full:
return self._drop(pending)
self._unreported_drops = 0
return True
+ def _write_sidecar(
+ self,
+ *,
+ encoded_records: int,
+ written_records: int,
+ dropped_records: int,
+ last_step_index: int | None,
+ final: bool,
+ ) -> None:
+ sidecar = dict(
+ schema=SCHEMA_VERSION,
+ record_type="footer_checkpoint",
+ stream=self.path.name,
+ encoded_records=encoded_records,
+ written_records=written_records,
+ dropped_records=dropped_records,
+ last_step_index=last_step_index,
+ checkpoint_wall_ns=time.time_ns(),
+ flush_interval_seconds=FLUSH_INTERVAL_SECONDS,
+ final=final,
+ )
+ temporary_path = self.sidecar_path.with_name(
+ f".{self.sidecar_path.name}.tmp-{os.getpid()}-{threading.get_ident()}"
+ )
+ try:
+ with temporary_path.open("wb") as temporary_file:
+ temporary_file.write(self._encoder.encode(sidecar) + b"\n")
+ temporary_file.flush()
+ os.replace(temporary_path, self.sidecar_path)
+ finally:
+ with suppress(FileNotFoundError):
+ temporary_path.unlink()
+
+ def _flush_checkpoint(self) -> None:
+ self._file.flush()
+ self._write_sidecar(
+ encoded_records=self._checkpoint_encoded_records,
+ written_records=self.written_records,
+ dropped_records=self._checkpoint_dropped_records,
+ last_step_index=self._last_written_step_index,
+ final=False,
+ )
+
def _run(self) -> None:
buffered = 0
last_flush = time.monotonic()
try:
while True:
try:
- item = self._queue.get(timeout=1.0)
+ item = self._queue.get(timeout=FLUSH_INTERVAL_SECONDS)
except queue.Empty:
- self._file.flush()
+ self._flush_checkpoint()
buffered = 0
last_flush = time.monotonic()
continue
try:
if item is _STOP:
break
- self._file.write(item)
- buffered += len(item)
+ payload, encoded, dropped, step_index = item
+ self._file.write(payload)
+ buffered += len(payload)
self.written_records += 1
+ self._checkpoint_encoded_records = encoded
+ self._checkpoint_dropped_records = dropped
+ self._last_written_step_index = step_index
now = time.monotonic()
- if buffered >= 1 << 20 or now - last_flush >= 1.0:
- self._file.flush()
+ if (
+ buffered >= 1 << 20
+ or now - last_flush >= FLUSH_INTERVAL_SECONDS
+ ):
+ self._flush_checkpoint()
buffered = 0
last_flush = now
finally:
@@ -163,6 +226,13 @@ class JSONLWriter:
try:
self._file.write(self._encoder.encode(footer) + b"\n")
self._file.flush()
+ self._write_sidecar(
+ encoded_records=footer["encoded_records"],
+ written_records=footer["written_records"],
+ dropped_records=footer["dropped_records"],
+ last_step_index=self._last_written_step_index,
+ final=True,
+ )
except Exception as error:
self._record_failure(error)
finally:
--
2.43.0

27
patches/vllm-0.24.0-opprof/0007-Recreate-scheduled-torch-profiler-between-windows.patch Normal file
View File

@@ -0,0 +1,27 @@
From 23450fb21ac255b0cf710f4ee965ee694921975d Mon Sep 17 00:00:00 2001
From: Gahow Wang <gahow.wang@gmail.com>
Date: Sun, 12 Jul 2026 13:12:52 +0800
Subject: [PATCH] Recreate scheduled torch profiler between windows
---
vllm/v1/worker/gpu_worker.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py
index 5e266a3..0058f96 100644
--- a/vllm/v1/worker/gpu_worker.py
+++ b/vllm/v1/worker/gpu_worker.py
@@ -978,6 +978,10 @@ class Worker(WorkerBase):
logger.warning("Profiler was not started, nothing to stop.")
return
self.profiler.stop()
+ # A scheduled torch.profiler.profile does not reset its schedule
+ # after stop(). Recreate it for the next /start_profile window.
+ if isinstance(self.profiler, TorchProfilerWrapper):
+ self.profiler = None
def execute_dummy_batch(self) -> None:
num_tokens = getattr(self.model_runner, "uniform_decode_query_len", 1)
--
2.43.0

119
patches/vllm-0.24.0-opprof/README.md Normal file
View File

@@ -0,0 +1,119 @@
# vLLM 0.24.0 OpProf patch series
## Goal
Apply the accepted OpProf Layer-1 instrumentation to exactly vLLM `v0.24.0`
at base commit `ee0da84ab9e04ac7610e28580af62c365e898389`. The series adds one
scheduler-owned composition record per step without installing new runtime
dependencies or changing GPU kernels.
## Contents
- `0001-Add-lightweight-per-step-OpProf-telemetry.patch`: adds the environment
switch, import-light JSONL recorder/writer, scheduler hooks, and reuse of the
existing CUDA-graph stat. Writer failures are exposed without blocking
producers or shutdown, and request histograms are accumulated in-place.
- `0002-Add-standalone-OpProf-telemetry-tests.patch`: adds CPU-only tests that
load the recorder directly without importing or installing vLLM or torch,
including ENOSPC, golden-record, and zero-token regressions.
- `0003-Log-the-OpProf-output-path-at-startup.patch`: logs the resolved JSONL
output path and covers it in the standalone shutdown test.
- `0004-Exclude-OpProf-output-path-from-compile-cache-key.patch`: prevents the
per-run telemetry destination from invalidating vLLM's torch.compile/AOT
cache and adds an import-light regression test.
- `0005-Keep-compile-factor-regression-import-light.patch`: isolates the new
regression from torch in full vLLM test environments.
- `0006-Checkpoint-OpProf-accounting-across-hard-kills.patch`: atomically
checkpoints balanced writer counters beside each JSONL stream once per
flush interval, with clean-close and hard-kill regressions.
- `0007-Recreate-scheduled-torch-profiler-between-windows.patch`: discards a
stopped scheduled torch-profiler wrapper so each subsequent official profile
endpoint call receives a fresh 2+8 schedule and emits its own trace.
- `apply.sh`: verifies the exact base, refuses dirty/wrong revisions, applies
all numbered patches with `git am`, and exits successfully only when the
exact series is already applied directly on the required base.
- `pytest-evidence.txt`: exact isolated test command, dependency versions, and
all-pass summary.
The source branch tip used to generate the patches is
`23450fb21ac255b0cf710f4ee965ee694921975d` (`opprof`).
## Apply
Prerequisite: a clean checkout whose `HEAD` is the exact base commit.
```bash
./patches/vllm-0.24.0-opprof/apply.sh /path/to/vllm-v0.24.0
```
Running the command again is a no-op only when the five matching patch commits
are rooted directly at the required base. A partially applied series, dirty
tree, unrelated commit, or any other `HEAD` is rejected instead of being
guessed around.
## Enable and output
Set an absolute output directory before starting vLLM:
```bash
export VLLM_OPPROF_DIR=/absolute/path/to/run/opprof
```
Unset or empty disables the feature before recorder construction. Combining it
with `--disable-log-stats` fails fast, as approved.
Each EngineCore/DP scheduler writes one file named approximately
`opprof-v1-dp0-pid1234-<start_ns>.jsonl`. Records contain schema/engine/step and
timestamps; scheduled prefill/decode composition; first/middle/final/unsplit
prefill chunks; 12-bin context and 9-bin chunk-size histograms; preemptions;
running/waiting/deferred queues; KV blocks/usage; local/external prefix deltas;
CUDA-graph hit/mode/bucket/padding; explicit null Layer-1 MoE load; and drop-gap
accounting. A clean close writes a final writer-count footer in the stream.
Every JSONL flush also atomically replaces
`<stream>.footer.json` through a same-directory temporary file. The sidecar
contains the encoded, written, and dropped counts through that durable flush,
the last written step index, a wall-clock timestamp, the one-second flush
interval, and whether it is final. Queue entries carry their submission
ordinal and cumulative drops, so a periodic checkpoint always satisfies
`encoded = written + dropped` without decoding records in the writer thread.
On clean close the in-stream footer is authoritative and the final sidecar must
agree with all three counters. If a hard kill prevents the in-stream footer,
the latest sidecar is authoritative: the decoded data-line count must equal
its `written_records`, its final data-line step must equal
`last_step_index`, and its counters must balance. Data after that checkpoint
may be lost, bounded by at most the configured one-second flush interval.
The bounded queue holds 8192 encoded records. Producers never wait for disk;
full queues or a failed writer drop the new record and report the gap on the
next successful record. A writer I/O failure is exposed through recorder state,
logged once, and cannot make shutdown wait indefinitely. The writer flushes at
1 MiB, one second, or shutdown.
## Test
Only pytest and msgspec are required. `--confcutdir` prevents vLLM's global
test configuration from importing its full dependency stack.
```bash
cd /path/to/vllm-v0.24.0
uv run --no-project --with pytest --with msgspec \
pytest --confcutdir=tests/v1/core tests/v1/core/test_opprof.py -q
```
Expected summary:
```text
18 passed in 1.09s
```
## Caveats
- Layer 1 intentionally records no expert-load arrays. Exact routed experts
remain a separate Layer-2 run.
- `PIECEWISE` means graph-wrapped compiled regions, not full-step graph replay.
- Phase 2 must measure the always-on overhead; acceptance requires the upper
bound of the 95% confidence interval to remain below 3% for every primary
serving metric.
- Primary campaign topology is TP1 on community BF16 Qwen3-30B-A3B, with TP2
and TP4 counterpoints. Record the selected MoE backend log every run.

58
patches/vllm-0.24.0-opprof/apply.sh Executable file
View File

@@ -0,0 +1,58 @@
#!/usr/bin/env bash
set -euo pipefail
base_commit=ee0da84ab9e04ac7610e28580af62c365e898389
script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
repo=${1:-.}
patches=("$script_dir"/0*.patch)
git -C "$repo" rev-parse --git-dir >/dev/null
if [[ -n $(git -C "$repo" status --porcelain) ]]; then
echo "Refusing to apply to a dirty worktree." >&2
exit 1
fi
if ((${#patches[@]} == 0)) || [[ ! -f ${patches[0]} ]]; then
echo "No numbered patch files found in $script_dir" >&2
exit 1
fi
head_commit=$(git -C "$repo" rev-parse HEAD)
mapfile -t recent_commits < <(
git -C "$repo" rev-list --max-count="${#patches[@]}" --reverse HEAD
)
patches_match=true
((${#recent_commits[@]} == ${#patches[@]})) || patches_match=false
for i in "${!patches[@]}"; do
$patches_match || break
patch_id=$(git patch-id --stable <"${patches[$i]}" | awk '{print $1}')
commit_id=$(
git -C "$repo" show --pretty=format: "${recent_commits[$i]}" |
git patch-id --stable | awk '{print $1}'
)
[[ $patch_id == "$commit_id" ]] || patches_match=false
done
first_parent=""
if ((${#recent_commits[@]} > 0)); then
first_parent=$(
git -C "$repo" rev-parse --verify "${recent_commits[0]}^" 2>/dev/null || true
)
fi
if $patches_match && [[ $first_parent == "$base_commit" ]]; then
echo "OpProf patch series is already applied."
exit 0
fi
if [[ $head_commit == "$base_commit" ]]; then
git -C "$repo" am "${patches[@]}"
echo "Applied OpProf patch series to $repo"
exit 0
fi
if $patches_match; then
echo "Refusing to treat OpProf patches as already applied:" >&2
echo "first patch parent is $first_parent, expected $base_commit" >&2
exit 1
fi
echo "Refusing to apply: HEAD is $head_commit; expected $base_commit" >&2
echo "or the exact OpProf patch series rooted at that base." >&2
exit 1

16
patches/vllm-0.24.0-opprof/pytest-evidence.txt Normal file
View File

@@ -0,0 +1,16 @@
OpProf standalone pytest evidence
Date: 2026-07-12
Source branch: opprof
Source tip: 23450fb21ac255b0cf710f4ee965ee694921975d
Base: ee0da84ab9e04ac7610e28580af62c365e898389 (v0.24.0)
Environment: Python 3.11.13, pytest 9.1.1, msgspec 0.21.1
vLLM installed: no
torch installed in isolated test environment: no
GPU/remote access: no
Command:
uv run --no-project --with pytest --with msgspec pytest --confcutdir=tests/v1/core tests/v1/core/test_opprof.py -q
Output:
.................. [100%]
18 passed in 1.09s