From 4f4ee674f217698436b00c3ab6357f59a792477a Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 11 Jul 2026 17:29:02 +0800 Subject: [PATCH 2/5] Add standalone OpProf telemetry tests Assisted-by: OpenAI Codex --- tests/v1/core/test_opprof.py | 397 +++++++++++++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 tests/v1/core/test_opprof.py diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py new file mode 100644 index 0000000..9bfbfcc --- /dev/null +++ b/tests/v1/core/test_opprof.py @@ -0,0 +1,397 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Standalone tests: this file intentionally does not import the vllm package.""" + +import errno +import importlib.util +import logging +import sys +import threading +from pathlib import Path +from types import SimpleNamespace + +import msgspec +import pytest + +_ROOT = Path(__file__).parents[3] +_SPEC = importlib.util.spec_from_file_location( + "opprof_standalone", _ROOT / "vllm" / "v1" / "opprof.py" +) +assert _SPEC is not None and _SPEC.loader is not None +opprof = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = opprof +_SPEC.loader.exec_module(opprof) + + +class CachedRequests: + def __init__(self, context_ids=()): + self.context_ids = set(context_ids) + + def is_context_phase(self, req_id): + return req_id in self.context_ids + + +def prefix_stats(**overrides): + values = dict.fromkeys(opprof._PREFIX_FIELDS, 0) + values.update(overrides) + return SimpleNamespace(**values) + + +def request(computed, total, was_chunk=False, placeholders=0): + return SimpleNamespace( + num_computed_tokens=computed, + num_tokens=total, + num_output_placeholders=placeholders, + is_prefill_chunk=was_chunk, + ) + + +def scheduler(requests, local=None, external=None): + block_pool = SimpleNamespace( + num_gpu_blocks=101, + get_num_free_blocks=lambda: 40, + ) + kv_manager = SimpleNamespace( + prefix_cache_stats=local or prefix_stats(), + block_pool=block_pool, + usage=0.6, + ) + return SimpleNamespace( + requests=requests, + kv_cache_manager=kv_manager, + connector_prefix_cache_stats=external, + running=list(range(len(requests))), + waiting=[0, 1], + skipped_waiting=[0], + ) + + +def schedule_output(tokens, context_ids=(), new_ids=(), preempted=()): + return SimpleNamespace( + scheduled_new_reqs=[SimpleNamespace(req_id=req_id) for req_id in new_ids], + scheduled_cached_reqs=CachedRequests(context_ids), + num_scheduled_tokens=tokens, + total_num_scheduled_tokens=sum(tokens.values()), + preempted_req_ids=set(preempted), + ) + + +def graph(mode="FULL", unpadded=1, padded=1): + return SimpleNamespace( + runtime_mode=mode, + num_unpadded_tokens=unpadded, + num_padded_tokens=padded, + num_paddings=padded - unpadded, + ) + + +def recorder(tmp_path, *, capacity=8192, start=True): + path = tmp_path / "opprof.jsonl" + writer = opprof.JSONLWriter(path, capacity=capacity, start=start) + return opprof.OpProfRecorder("dp0-pid1", writer), path + + +def emit(rec, sched, output, cg=None): + start = rec.capture_start(sched) + rec.begin(sched, output, start) + return rec.finalize(output, cg or graph()) + + +def read_jsonl(path): + return [msgspec.json.decode(line) for line in path.read_bytes().splitlines()] + + +def test_import_light_and_approved_constants(): + assert "torch" not in sys.modules + assert "vllm" not in sys.modules + assert opprof.DEFAULT_QUEUE_CAPACITY == 8192 + assert tuple(1 << i for i in range(7, 18)) == opprof.CONTEXT_LENGTH_EDGES + assert tuple(1 << i for i in range(4, 12)) == opprof.CHUNK_SIZE_EDGES + + +def test_schema_and_invariants(tmp_path): + sched = scheduler( + { + "first": request(0, 100), + "final": request(64, 100, was_chunk=True), + "decode": request(1024, 1025), + } + ) + output = schedule_output( + {"first": 64, "final": 36, "decode": 1}, + context_ids={"final"}, + new_ids={"first"}, + preempted={"old"}, + ) + rec, path = recorder(tmp_path) + start = rec.capture_start(sched) + sched.kv_cache_manager.prefix_cache_stats = prefix_stats( + requests=1, queries=100, hits=64 + ) + rec.begin(sched, output, start) + assert rec.finalize(output, graph("FULL", 101, 128)) + rec.close() + + record, footer = read_jsonl(path) + assert record["schema"] == 1 + assert record["scheduled_requests"] == 3 + assert record["prefill_requests"] == 2 + assert record["decode_batch_size"] == 1 + assert record["prefill_tokens"] + record["decode_tokens"] == 101 + assert sum(record["context_length_hist"]) == 3 + assert len(record["context_length_hist"]) == 12 + assert sum(record["chunked_prefill"]["chunk_size_hist"]) == 2 + assert len(record["chunked_prefill"]["chunk_size_hist"]) == 9 + assert record["chunked_prefill"]["first"] == 1 + assert record["chunked_prefill"]["final"] == 1 + assert record["preemptions"] == 1 + assert record["kv"] == { + "total_blocks": 100, + "free_blocks": 40, + "used_blocks": 60, + "usage": 0.6, + } + assert record["prefix"]["local"]["hits"] == 64 + assert record["moe_expert_load"] is None + assert record["complete_mono_ns"] >= record["submit_mono_ns"] + assert footer["record_type"] == "footer" + assert footer["written_records"] == 1 + + +def test_capture_record_matches_pre_refactor_golden(tmp_path, monkeypatch): + sched = scheduler( + { + "edge": request(0, 128), + "after": request(65, 129, was_chunk=True), + "decode": request(256, 257), + } + ) + output = schedule_output( + {"edge": 128, "after": 64, "decode": 1}, + context_ids={"after"}, + new_ids={"edge"}, + preempted={"old"}, + ) + rec, path = recorder(tmp_path) + zero_prefix = dict.fromkeys(opprof._PREFIX_FIELDS, 0) + start = (100, 200, {"local": zero_prefix, "external": None}) + monkeypatch.setattr(opprof.time, "monotonic_ns", lambda: 300) + + rec.begin(sched, output, start) + assert rec.finalize(output, graph("FULL", 193, 256)) + rec.close() + + record = read_jsonl(path)[0] + assert record == { + "schema": 1, + "engine_id": "dp0-pid1", + "step_index": 0, + "submit_wall_ns": 100, + "submit_mono_ns": 200, + "model_executed": True, + "scheduled_requests": 3, + "decode_batch_size": 1, + "prefill_requests": 2, + "prefill_tokens": 192, + "decode_tokens": 1, + "chunked_prefill": { + "first": 0, + "middle": 0, + "final": 1, + "unsplit": 1, + "tokens": 192, + "chunk_size_hist": [0, 0, 1, 1, 0, 0, 0, 0, 0], + }, + "context_length_hist": [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "preemptions": 1, + "queues": {"running": 3, "waiting": 2, "deferred": 1}, + "kv": { + "total_blocks": 100, + "free_blocks": 40, + "used_blocks": 60, + "usage": 0.6, + }, + "prefix": {"local": zero_prefix, "external": None}, + "complete_mono_ns": 300, + "cudagraph": { + "hit": True, + "runtime_mode": "FULL", + "unpadded_tokens": 193, + "bucket_tokens": 256, + "padding_tokens": 63, + }, + "moe_expert_load": None, + "dropped_records_before": 0, + } + + +@pytest.mark.parametrize( + ("was_chunk", "end", "target", "expected"), + [ + (False, 64, 100, "first"), + (True, 80, 100, "middle"), + (True, 100, 100, "final"), + (False, 100, 100, "unsplit"), + ], +) +def test_chunk_classification(was_chunk, end, target, expected): + assert opprof.classify_chunk(was_chunk, end, target) == expected + + +def test_async_pairing_out_of_order_and_double_finalize(tmp_path): + sched = scheduler({"a": request(10, 11), "b": request(200, 201)}) + first = schedule_output({"a": 1}) + second = schedule_output({"b": 1}) + rec, path = recorder(tmp_path) + rec.begin(sched, first, rec.capture_start(sched)) + rec.begin(sched, second, rec.capture_start(sched)) + assert rec.finalize(second, graph()) + assert rec.finalize(first, graph("NONE")) + with pytest.raises(AssertionError, match="already finalized"): + rec.finalize(second, graph()) + assert not rec._pending + rec.close() + records = read_jsonl(path)[:-1] + assert [record["step_index"] for record in records] == [1, 0] + assert records[0]["context_length_hist"][1] == 1 + assert records[1]["context_length_hist"][0] == 1 + + +def test_disabled_noop_and_log_stats_fail_fast(tmp_path): + assert opprof.OpProfRecorder.create("", dp_rank=0, log_stats=False) is None + with pytest.raises(ValueError, match="requires log stats"): + opprof.OpProfRecorder.create(str(tmp_path), dp_rank=0, log_stats=False) + assert not list(tmp_path.iterdir()) + + +def test_bounded_queue_drop_accounting(tmp_path): + sched = scheduler({str(i): request(i, i + 1) for i in range(3)}) + rec, path = recorder(tmp_path, capacity=1, start=False) + assert emit(rec, sched, schedule_output({"0": 1})) + assert not emit(rec, sched, schedule_output({"1": 1})) + rec.writer.start() + rec.writer._queue.join() + assert emit(rec, sched, schedule_output({"2": 1})) + rec.close() + + first, after_drop, footer = read_jsonl(path) + assert first["step_index"] == 0 + assert after_drop["step_index"] == 2 + assert after_drop["dropped_records_before"] == 1 + assert footer["encoded_records"] == 3 + assert footer["written_records"] == 2 + assert footer["dropped_records"] == 1 + + +def test_writer_enospc_is_exposed_and_shutdown_is_bounded( + tmp_path, monkeypatch, caplog +): + sched = scheduler( + { + "first": request(0, 1), + "after_failure": request(1, 2), + } + ) + rec, _ = recorder(tmp_path, capacity=1, start=False) + assert emit(rec, sched, schedule_output({"first": 1})) + + real_file = rec.writer._file + + def fail_enospc(*_args, **_kwargs): + raise OSError(errno.ENOSPC, "No space left on device") + + failing_file = SimpleNamespace( + write=fail_enospc, + flush=fail_enospc, + close=real_file.close, + ) + monkeypatch.setattr(rec.writer, "_file", failing_file) + caplog.set_level(logging.ERROR, logger=opprof.__name__) + + rec.writer.start() + rec.writer._thread.join(timeout=1.0) + assert not rec.writer._thread.is_alive() + + producer_result = emit( + rec, sched, schedule_output({"after_failure": 1}) + ) + + closer = threading.Thread(target=rec.close, daemon=True) + closer.start() + closer.join(timeout=1.0) + assert not closer.is_alive(), "OpProf close blocked after writer failure" + assert not producer_result + assert rec.writer.dropped_records == 1 + assert rec.failed + assert isinstance(rec.failure, OSError) + assert rec.failure.errno == errno.ENOSPC + errors = [ + record + for record in caplog.records + if "OpProf writer failed" in record.getMessage() + ] + assert len(errors) == 1 + + +def test_shutdown_flush_is_idempotent(tmp_path): + sched = scheduler({"decode": request(8, 9)}) + rec, path = recorder(tmp_path) + assert emit(rec, sched, schedule_output({"decode": 1})) + rec.close() + rec.close() + record, footer = read_jsonl(path) + assert record["step_index"] == 0 + assert footer["written_records"] == 1 + assert path.stat().st_size > 0 + + +def test_piecewise_cudagraph_record_preserved(tmp_path): + sched = scheduler({"decode": request(4096, 4097)}) + output = schedule_output({"decode": 1}) + rec, path = recorder(tmp_path) + assert emit(rec, sched, output, graph("PIECEWISE", 513, 520)) + rec.close() + record = read_jsonl(path)[0] + assert record["cudagraph"] == { + "hit": True, + "runtime_mode": "PIECEWISE", + "unpadded_tokens": 513, + "bucket_tokens": 520, + "padding_tokens": 7, + } + + +def test_zero_scheduled_tokens_finalize_without_cudagraph(tmp_path): + sched = scheduler({}) + output = schedule_output({}) + rec, path = recorder(tmp_path) + rec.begin(sched, output, rec.capture_start(sched)) + assert rec._pending + + assert rec.finalize(output, cudagraph_stat=None) + assert not rec._pending + rec.close() + + record = read_jsonl(path)[0] + assert record["model_executed"] is False + assert record["scheduled_requests"] == 0 + assert record["prefill_requests"] == 0 + assert record["prefill_tokens"] == 0 + assert record["decode_batch_size"] == 0 + assert record["decode_tokens"] == 0 + assert record["context_length_hist"] == [0] * 12 + assert record["chunked_prefill"] == { + "first": 0, + "middle": 0, + "final": 0, + "unsplit": 0, + "tokens": 0, + "chunk_size_hist": [0] * 9, + } + assert record["cudagraph"] == { + "hit": False, + "runtime_mode": "NONE", + "unpadded_tokens": 0, + "bucket_tokens": 0, + "padding_tokens": 0, + } -- 2.43.0