Gahow Wang
d5b276180d
Workload-conditioned operator profiling on patched vLLM 0.24.0 + Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64 raggedness, 8-45% token-efficiency loss vs rectangular controls); mechanism decomposition kills the padding narrative and finds the arrival-uniformization artifact (-12.9%); cross-version churn surface shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held. Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer sidecars and metrics are tracked. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
418 lines
13 KiB
Diff
418 lines
13 KiB
Diff
From 4f4ee674f217698436b00c3ab6357f59a792477a Mon Sep 17 00:00:00 2001
|
|
From: Gahow Wang <gahow.wang@gmail.com>
|
|
Date: Sat, 11 Jul 2026 17:29:02 +0800
|
|
Subject: [PATCH 2/5] Add standalone OpProf telemetry tests
|
|
|
|
Assisted-by: OpenAI Codex
|
|
---
|
|
tests/v1/core/test_opprof.py | 397 +++++++++++++++++++++++++++++++++++
|
|
1 file changed, 397 insertions(+)
|
|
create mode 100644 tests/v1/core/test_opprof.py
|
|
|
|
diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
|
|
new file mode 100644
|
|
index 0000000..9bfbfcc
|
|
--- /dev/null
|
|
+++ b/tests/v1/core/test_opprof.py
|
|
@@ -0,0 +1,397 @@
|
|
+# SPDX-License-Identifier: Apache-2.0
|
|
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
+"""Standalone tests: this file intentionally does not import the vllm package."""
|
|
+
|
|
+import errno
|
|
+import importlib.util
|
|
+import logging
|
|
+import sys
|
|
+import threading
|
|
+from pathlib import Path
|
|
+from types import SimpleNamespace
|
|
+
|
|
+import msgspec
|
|
+import pytest
|
|
+
|
|
+_ROOT = Path(__file__).parents[3]
|
|
+_SPEC = importlib.util.spec_from_file_location(
|
|
+ "opprof_standalone", _ROOT / "vllm" / "v1" / "opprof.py"
|
|
+)
|
|
+assert _SPEC is not None and _SPEC.loader is not None
|
|
+opprof = importlib.util.module_from_spec(_SPEC)
|
|
+sys.modules[_SPEC.name] = opprof
|
|
+_SPEC.loader.exec_module(opprof)
|
|
+
|
|
+
|
|
+class CachedRequests:
|
|
+ def __init__(self, context_ids=()):
|
|
+ self.context_ids = set(context_ids)
|
|
+
|
|
+ def is_context_phase(self, req_id):
|
|
+ return req_id in self.context_ids
|
|
+
|
|
+
|
|
+def prefix_stats(**overrides):
|
|
+ values = dict.fromkeys(opprof._PREFIX_FIELDS, 0)
|
|
+ values.update(overrides)
|
|
+ return SimpleNamespace(**values)
|
|
+
|
|
+
|
|
+def request(computed, total, was_chunk=False, placeholders=0):
|
|
+ return SimpleNamespace(
|
|
+ num_computed_tokens=computed,
|
|
+ num_tokens=total,
|
|
+ num_output_placeholders=placeholders,
|
|
+ is_prefill_chunk=was_chunk,
|
|
+ )
|
|
+
|
|
+
|
|
+def scheduler(requests, local=None, external=None):
|
|
+ block_pool = SimpleNamespace(
|
|
+ num_gpu_blocks=101,
|
|
+ get_num_free_blocks=lambda: 40,
|
|
+ )
|
|
+ kv_manager = SimpleNamespace(
|
|
+ prefix_cache_stats=local or prefix_stats(),
|
|
+ block_pool=block_pool,
|
|
+ usage=0.6,
|
|
+ )
|
|
+ return SimpleNamespace(
|
|
+ requests=requests,
|
|
+ kv_cache_manager=kv_manager,
|
|
+ connector_prefix_cache_stats=external,
|
|
+ running=list(range(len(requests))),
|
|
+ waiting=[0, 1],
|
|
+ skipped_waiting=[0],
|
|
+ )
|
|
+
|
|
+
|
|
+def schedule_output(tokens, context_ids=(), new_ids=(), preempted=()):
|
|
+ return SimpleNamespace(
|
|
+ scheduled_new_reqs=[SimpleNamespace(req_id=req_id) for req_id in new_ids],
|
|
+ scheduled_cached_reqs=CachedRequests(context_ids),
|
|
+ num_scheduled_tokens=tokens,
|
|
+ total_num_scheduled_tokens=sum(tokens.values()),
|
|
+ preempted_req_ids=set(preempted),
|
|
+ )
|
|
+
|
|
+
|
|
+def graph(mode="FULL", unpadded=1, padded=1):
|
|
+ return SimpleNamespace(
|
|
+ runtime_mode=mode,
|
|
+ num_unpadded_tokens=unpadded,
|
|
+ num_padded_tokens=padded,
|
|
+ num_paddings=padded - unpadded,
|
|
+ )
|
|
+
|
|
+
|
|
+def recorder(tmp_path, *, capacity=8192, start=True):
|
|
+ path = tmp_path / "opprof.jsonl"
|
|
+ writer = opprof.JSONLWriter(path, capacity=capacity, start=start)
|
|
+ return opprof.OpProfRecorder("dp0-pid1", writer), path
|
|
+
|
|
+
|
|
+def emit(rec, sched, output, cg=None):
|
|
+ start = rec.capture_start(sched)
|
|
+ rec.begin(sched, output, start)
|
|
+ return rec.finalize(output, cg or graph())
|
|
+
|
|
+
|
|
+def read_jsonl(path):
|
|
+ return [msgspec.json.decode(line) for line in path.read_bytes().splitlines()]
|
|
+
|
|
+
|
|
+def test_import_light_and_approved_constants():
|
|
+ assert "torch" not in sys.modules
|
|
+ assert "vllm" not in sys.modules
|
|
+ assert opprof.DEFAULT_QUEUE_CAPACITY == 8192
|
|
+ assert tuple(1 << i for i in range(7, 18)) == opprof.CONTEXT_LENGTH_EDGES
|
|
+ assert tuple(1 << i for i in range(4, 12)) == opprof.CHUNK_SIZE_EDGES
|
|
+
|
|
+
|
|
+def test_schema_and_invariants(tmp_path):
|
|
+ sched = scheduler(
|
|
+ {
|
|
+ "first": request(0, 100),
|
|
+ "final": request(64, 100, was_chunk=True),
|
|
+ "decode": request(1024, 1025),
|
|
+ }
|
|
+ )
|
|
+ output = schedule_output(
|
|
+ {"first": 64, "final": 36, "decode": 1},
|
|
+ context_ids={"final"},
|
|
+ new_ids={"first"},
|
|
+ preempted={"old"},
|
|
+ )
|
|
+ rec, path = recorder(tmp_path)
|
|
+ start = rec.capture_start(sched)
|
|
+ sched.kv_cache_manager.prefix_cache_stats = prefix_stats(
|
|
+ requests=1, queries=100, hits=64
|
|
+ )
|
|
+ rec.begin(sched, output, start)
|
|
+ assert rec.finalize(output, graph("FULL", 101, 128))
|
|
+ rec.close()
|
|
+
|
|
+ record, footer = read_jsonl(path)
|
|
+ assert record["schema"] == 1
|
|
+ assert record["scheduled_requests"] == 3
|
|
+ assert record["prefill_requests"] == 2
|
|
+ assert record["decode_batch_size"] == 1
|
|
+ assert record["prefill_tokens"] + record["decode_tokens"] == 101
|
|
+ assert sum(record["context_length_hist"]) == 3
|
|
+ assert len(record["context_length_hist"]) == 12
|
|
+ assert sum(record["chunked_prefill"]["chunk_size_hist"]) == 2
|
|
+ assert len(record["chunked_prefill"]["chunk_size_hist"]) == 9
|
|
+ assert record["chunked_prefill"]["first"] == 1
|
|
+ assert record["chunked_prefill"]["final"] == 1
|
|
+ assert record["preemptions"] == 1
|
|
+ assert record["kv"] == {
|
|
+ "total_blocks": 100,
|
|
+ "free_blocks": 40,
|
|
+ "used_blocks": 60,
|
|
+ "usage": 0.6,
|
|
+ }
|
|
+ assert record["prefix"]["local"]["hits"] == 64
|
|
+ assert record["moe_expert_load"] is None
|
|
+ assert record["complete_mono_ns"] >= record["submit_mono_ns"]
|
|
+ assert footer["record_type"] == "footer"
|
|
+ assert footer["written_records"] == 1
|
|
+
|
|
+
|
|
+def test_capture_record_matches_pre_refactor_golden(tmp_path, monkeypatch):
|
|
+ sched = scheduler(
|
|
+ {
|
|
+ "edge": request(0, 128),
|
|
+ "after": request(65, 129, was_chunk=True),
|
|
+ "decode": request(256, 257),
|
|
+ }
|
|
+ )
|
|
+ output = schedule_output(
|
|
+ {"edge": 128, "after": 64, "decode": 1},
|
|
+ context_ids={"after"},
|
|
+ new_ids={"edge"},
|
|
+ preempted={"old"},
|
|
+ )
|
|
+ rec, path = recorder(tmp_path)
|
|
+ zero_prefix = dict.fromkeys(opprof._PREFIX_FIELDS, 0)
|
|
+ start = (100, 200, {"local": zero_prefix, "external": None})
|
|
+ monkeypatch.setattr(opprof.time, "monotonic_ns", lambda: 300)
|
|
+
|
|
+ rec.begin(sched, output, start)
|
|
+ assert rec.finalize(output, graph("FULL", 193, 256))
|
|
+ rec.close()
|
|
+
|
|
+ record = read_jsonl(path)[0]
|
|
+ assert record == {
|
|
+ "schema": 1,
|
|
+ "engine_id": "dp0-pid1",
|
|
+ "step_index": 0,
|
|
+ "submit_wall_ns": 100,
|
|
+ "submit_mono_ns": 200,
|
|
+ "model_executed": True,
|
|
+ "scheduled_requests": 3,
|
|
+ "decode_batch_size": 1,
|
|
+ "prefill_requests": 2,
|
|
+ "prefill_tokens": 192,
|
|
+ "decode_tokens": 1,
|
|
+ "chunked_prefill": {
|
|
+ "first": 0,
|
|
+ "middle": 0,
|
|
+ "final": 1,
|
|
+ "unsplit": 1,
|
|
+ "tokens": 192,
|
|
+ "chunk_size_hist": [0, 0, 1, 1, 0, 0, 0, 0, 0],
|
|
+ },
|
|
+ "context_length_hist": [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
|
|
+ "preemptions": 1,
|
|
+ "queues": {"running": 3, "waiting": 2, "deferred": 1},
|
|
+ "kv": {
|
|
+ "total_blocks": 100,
|
|
+ "free_blocks": 40,
|
|
+ "used_blocks": 60,
|
|
+ "usage": 0.6,
|
|
+ },
|
|
+ "prefix": {"local": zero_prefix, "external": None},
|
|
+ "complete_mono_ns": 300,
|
|
+ "cudagraph": {
|
|
+ "hit": True,
|
|
+ "runtime_mode": "FULL",
|
|
+ "unpadded_tokens": 193,
|
|
+ "bucket_tokens": 256,
|
|
+ "padding_tokens": 63,
|
|
+ },
|
|
+ "moe_expert_load": None,
|
|
+ "dropped_records_before": 0,
|
|
+ }
|
|
+
|
|
+
|
|
+@pytest.mark.parametrize(
|
|
+ ("was_chunk", "end", "target", "expected"),
|
|
+ [
|
|
+ (False, 64, 100, "first"),
|
|
+ (True, 80, 100, "middle"),
|
|
+ (True, 100, 100, "final"),
|
|
+ (False, 100, 100, "unsplit"),
|
|
+ ],
|
|
+)
|
|
+def test_chunk_classification(was_chunk, end, target, expected):
|
|
+ assert opprof.classify_chunk(was_chunk, end, target) == expected
|
|
+
|
|
+
|
|
+def test_async_pairing_out_of_order_and_double_finalize(tmp_path):
|
|
+ sched = scheduler({"a": request(10, 11), "b": request(200, 201)})
|
|
+ first = schedule_output({"a": 1})
|
|
+ second = schedule_output({"b": 1})
|
|
+ rec, path = recorder(tmp_path)
|
|
+ rec.begin(sched, first, rec.capture_start(sched))
|
|
+ rec.begin(sched, second, rec.capture_start(sched))
|
|
+ assert rec.finalize(second, graph())
|
|
+ assert rec.finalize(first, graph("NONE"))
|
|
+ with pytest.raises(AssertionError, match="already finalized"):
|
|
+ rec.finalize(second, graph())
|
|
+ assert not rec._pending
|
|
+ rec.close()
|
|
+ records = read_jsonl(path)[:-1]
|
|
+ assert [record["step_index"] for record in records] == [1, 0]
|
|
+ assert records[0]["context_length_hist"][1] == 1
|
|
+ assert records[1]["context_length_hist"][0] == 1
|
|
+
|
|
+
|
|
+def test_disabled_noop_and_log_stats_fail_fast(tmp_path):
|
|
+ assert opprof.OpProfRecorder.create("", dp_rank=0, log_stats=False) is None
|
|
+ with pytest.raises(ValueError, match="requires log stats"):
|
|
+ opprof.OpProfRecorder.create(str(tmp_path), dp_rank=0, log_stats=False)
|
|
+ assert not list(tmp_path.iterdir())
|
|
+
|
|
+
|
|
+def test_bounded_queue_drop_accounting(tmp_path):
|
|
+ sched = scheduler({str(i): request(i, i + 1) for i in range(3)})
|
|
+ rec, path = recorder(tmp_path, capacity=1, start=False)
|
|
+ assert emit(rec, sched, schedule_output({"0": 1}))
|
|
+ assert not emit(rec, sched, schedule_output({"1": 1}))
|
|
+ rec.writer.start()
|
|
+ rec.writer._queue.join()
|
|
+ assert emit(rec, sched, schedule_output({"2": 1}))
|
|
+ rec.close()
|
|
+
|
|
+ first, after_drop, footer = read_jsonl(path)
|
|
+ assert first["step_index"] == 0
|
|
+ assert after_drop["step_index"] == 2
|
|
+ assert after_drop["dropped_records_before"] == 1
|
|
+ assert footer["encoded_records"] == 3
|
|
+ assert footer["written_records"] == 2
|
|
+ assert footer["dropped_records"] == 1
|
|
+
|
|
+
|
|
+def test_writer_enospc_is_exposed_and_shutdown_is_bounded(
|
|
+ tmp_path, monkeypatch, caplog
|
|
+):
|
|
+ sched = scheduler(
|
|
+ {
|
|
+ "first": request(0, 1),
|
|
+ "after_failure": request(1, 2),
|
|
+ }
|
|
+ )
|
|
+ rec, _ = recorder(tmp_path, capacity=1, start=False)
|
|
+ assert emit(rec, sched, schedule_output({"first": 1}))
|
|
+
|
|
+ real_file = rec.writer._file
|
|
+
|
|
+ def fail_enospc(*_args, **_kwargs):
|
|
+ raise OSError(errno.ENOSPC, "No space left on device")
|
|
+
|
|
+ failing_file = SimpleNamespace(
|
|
+ write=fail_enospc,
|
|
+ flush=fail_enospc,
|
|
+ close=real_file.close,
|
|
+ )
|
|
+ monkeypatch.setattr(rec.writer, "_file", failing_file)
|
|
+ caplog.set_level(logging.ERROR, logger=opprof.__name__)
|
|
+
|
|
+ rec.writer.start()
|
|
+ rec.writer._thread.join(timeout=1.0)
|
|
+ assert not rec.writer._thread.is_alive()
|
|
+
|
|
+ producer_result = emit(
|
|
+ rec, sched, schedule_output({"after_failure": 1})
|
|
+ )
|
|
+
|
|
+ closer = threading.Thread(target=rec.close, daemon=True)
|
|
+ closer.start()
|
|
+ closer.join(timeout=1.0)
|
|
+ assert not closer.is_alive(), "OpProf close blocked after writer failure"
|
|
+ assert not producer_result
|
|
+ assert rec.writer.dropped_records == 1
|
|
+ assert rec.failed
|
|
+ assert isinstance(rec.failure, OSError)
|
|
+ assert rec.failure.errno == errno.ENOSPC
|
|
+ errors = [
|
|
+ record
|
|
+ for record in caplog.records
|
|
+ if "OpProf writer failed" in record.getMessage()
|
|
+ ]
|
|
+ assert len(errors) == 1
|
|
+
|
|
+
|
|
+def test_shutdown_flush_is_idempotent(tmp_path):
|
|
+ sched = scheduler({"decode": request(8, 9)})
|
|
+ rec, path = recorder(tmp_path)
|
|
+ assert emit(rec, sched, schedule_output({"decode": 1}))
|
|
+ rec.close()
|
|
+ rec.close()
|
|
+ record, footer = read_jsonl(path)
|
|
+ assert record["step_index"] == 0
|
|
+ assert footer["written_records"] == 1
|
|
+ assert path.stat().st_size > 0
|
|
+
|
|
+
|
|
+def test_piecewise_cudagraph_record_preserved(tmp_path):
|
|
+ sched = scheduler({"decode": request(4096, 4097)})
|
|
+ output = schedule_output({"decode": 1})
|
|
+ rec, path = recorder(tmp_path)
|
|
+ assert emit(rec, sched, output, graph("PIECEWISE", 513, 520))
|
|
+ rec.close()
|
|
+ record = read_jsonl(path)[0]
|
|
+ assert record["cudagraph"] == {
|
|
+ "hit": True,
|
|
+ "runtime_mode": "PIECEWISE",
|
|
+ "unpadded_tokens": 513,
|
|
+ "bucket_tokens": 520,
|
|
+ "padding_tokens": 7,
|
|
+ }
|
|
+
|
|
+
|
|
+def test_zero_scheduled_tokens_finalize_without_cudagraph(tmp_path):
|
|
+ sched = scheduler({})
|
|
+ output = schedule_output({})
|
|
+ rec, path = recorder(tmp_path)
|
|
+ rec.begin(sched, output, rec.capture_start(sched))
|
|
+ assert rec._pending
|
|
+
|
|
+ assert rec.finalize(output, cudagraph_stat=None)
|
|
+ assert not rec._pending
|
|
+ rec.close()
|
|
+
|
|
+ record = read_jsonl(path)[0]
|
|
+ assert record["model_executed"] is False
|
|
+ assert record["scheduled_requests"] == 0
|
|
+ assert record["prefill_requests"] == 0
|
|
+ assert record["prefill_tokens"] == 0
|
|
+ assert record["decode_batch_size"] == 0
|
|
+ assert record["decode_tokens"] == 0
|
|
+ assert record["context_length_hist"] == [0] * 12
|
|
+ assert record["chunked_prefill"] == {
|
|
+ "first": 0,
|
|
+ "middle": 0,
|
|
+ "final": 0,
|
|
+ "unsplit": 0,
|
|
+ "tokens": 0,
|
|
+ "chunk_size_hist": [0] * 9,
|
|
+ }
|
|
+ assert record["cudagraph"] == {
|
|
+ "hit": False,
|
|
+ "runtime_mode": "NONE",
|
|
+ "unpadded_tokens": 0,
|
|
+ "bucket_tokens": 0,
|
|
+ "padding_tokens": 0,
|
|
+ }
|
|
--
|
|
2.43.0
|
|
|