From 4f4ee674f217698436b00c3ab6357f59a792477a Mon Sep 17 00:00:00 2001
From: Gahow Wang <gahow.wang@gmail.com>
Date: Sat, 11 Jul 2026 17:29:02 +0800
Subject: [PATCH 2/5] Add standalone OpProf telemetry tests

Assisted-by: OpenAI Codex
---
 tests/v1/core/test_opprof.py | 397 +++++++++++++++++++++++++++++++++++
 1 file changed, 397 insertions(+)
 create mode 100644 tests/v1/core/test_opprof.py

diff --git a/tests/v1/core/test_opprof.py b/tests/v1/core/test_opprof.py
new file mode 100644
index 0000000..9bfbfcc
--- /dev/null
+++ b/tests/v1/core/test_opprof.py
@@ -0,0 +1,397 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Standalone tests: this file intentionally does not import the vllm package."""
+
+import errno
+import importlib.util
+import logging
+import sys
+import threading
+from pathlib import Path
+from types import SimpleNamespace
+
+import msgspec
+import pytest
+
+_ROOT = Path(__file__).parents[3]
+_SPEC = importlib.util.spec_from_file_location(
+    "opprof_standalone", _ROOT / "vllm" / "v1" / "opprof.py"
+)
+assert _SPEC is not None and _SPEC.loader is not None
+opprof = importlib.util.module_from_spec(_SPEC)
+sys.modules[_SPEC.name] = opprof
+_SPEC.loader.exec_module(opprof)
+
+
+class CachedRequests:
+    def __init__(self, context_ids=()):
+        self.context_ids = set(context_ids)
+
+    def is_context_phase(self, req_id):
+        return req_id in self.context_ids
+
+
+def prefix_stats(**overrides):
+    values = dict.fromkeys(opprof._PREFIX_FIELDS, 0)
+    values.update(overrides)
+    return SimpleNamespace(**values)
+
+
+def request(computed, total, was_chunk=False, placeholders=0):
+    return SimpleNamespace(
+        num_computed_tokens=computed,
+        num_tokens=total,
+        num_output_placeholders=placeholders,
+        is_prefill_chunk=was_chunk,
+    )
+
+
+def scheduler(requests, local=None, external=None):
+    block_pool = SimpleNamespace(
+        num_gpu_blocks=101,
+        get_num_free_blocks=lambda: 40,
+    )
+    kv_manager = SimpleNamespace(
+        prefix_cache_stats=local or prefix_stats(),
+        block_pool=block_pool,
+        usage=0.6,
+    )
+    return SimpleNamespace(
+        requests=requests,
+        kv_cache_manager=kv_manager,
+        connector_prefix_cache_stats=external,
+        running=list(range(len(requests))),
+        waiting=[0, 1],
+        skipped_waiting=[0],
+    )
+
+
+def schedule_output(tokens, context_ids=(), new_ids=(), preempted=()):
+    return SimpleNamespace(
+        scheduled_new_reqs=[SimpleNamespace(req_id=req_id) for req_id in new_ids],
+        scheduled_cached_reqs=CachedRequests(context_ids),
+        num_scheduled_tokens=tokens,
+        total_num_scheduled_tokens=sum(tokens.values()),
+        preempted_req_ids=set(preempted),
+    )
+
+
+def graph(mode="FULL", unpadded=1, padded=1):
+    return SimpleNamespace(
+        runtime_mode=mode,
+        num_unpadded_tokens=unpadded,
+        num_padded_tokens=padded,
+        num_paddings=padded - unpadded,
+    )
+
+
+def recorder(tmp_path, *, capacity=8192, start=True):
+    path = tmp_path / "opprof.jsonl"
+    writer = opprof.JSONLWriter(path, capacity=capacity, start=start)
+    return opprof.OpProfRecorder("dp0-pid1", writer), path
+
+
+def emit(rec, sched, output, cg=None):
+    start = rec.capture_start(sched)
+    rec.begin(sched, output, start)
+    return rec.finalize(output, cg or graph())
+
+
+def read_jsonl(path):
+    return [msgspec.json.decode(line) for line in path.read_bytes().splitlines()]
+
+
+def test_import_light_and_approved_constants():
+    assert "torch" not in sys.modules
+    assert "vllm" not in sys.modules
+    assert opprof.DEFAULT_QUEUE_CAPACITY == 8192
+    assert tuple(1 << i for i in range(7, 18)) == opprof.CONTEXT_LENGTH_EDGES
+    assert tuple(1 << i for i in range(4, 12)) == opprof.CHUNK_SIZE_EDGES
+
+
+def test_schema_and_invariants(tmp_path):
+    sched = scheduler(
+        {
+            "first": request(0, 100),
+            "final": request(64, 100, was_chunk=True),
+            "decode": request(1024, 1025),
+        }
+    )
+    output = schedule_output(
+        {"first": 64, "final": 36, "decode": 1},
+        context_ids={"final"},
+        new_ids={"first"},
+        preempted={"old"},
+    )
+    rec, path = recorder(tmp_path)
+    start = rec.capture_start(sched)
+    sched.kv_cache_manager.prefix_cache_stats = prefix_stats(
+        requests=1, queries=100, hits=64
+    )
+    rec.begin(sched, output, start)
+    assert rec.finalize(output, graph("FULL", 101, 128))
+    rec.close()
+
+    record, footer = read_jsonl(path)
+    assert record["schema"] == 1
+    assert record["scheduled_requests"] == 3
+    assert record["prefill_requests"] == 2
+    assert record["decode_batch_size"] == 1
+    assert record["prefill_tokens"] + record["decode_tokens"] == 101
+    assert sum(record["context_length_hist"]) == 3
+    assert len(record["context_length_hist"]) == 12
+    assert sum(record["chunked_prefill"]["chunk_size_hist"]) == 2
+    assert len(record["chunked_prefill"]["chunk_size_hist"]) == 9
+    assert record["chunked_prefill"]["first"] == 1
+    assert record["chunked_prefill"]["final"] == 1
+    assert record["preemptions"] == 1
+    assert record["kv"] == {
+        "total_blocks": 100,
+        "free_blocks": 40,
+        "used_blocks": 60,
+        "usage": 0.6,
+    }
+    assert record["prefix"]["local"]["hits"] == 64
+    assert record["moe_expert_load"] is None
+    assert record["complete_mono_ns"] >= record["submit_mono_ns"]
+    assert footer["record_type"] == "footer"
+    assert footer["written_records"] == 1
+
+
+def test_capture_record_matches_pre_refactor_golden(tmp_path, monkeypatch):
+    sched = scheduler(
+        {
+            "edge": request(0, 128),
+            "after": request(65, 129, was_chunk=True),
+            "decode": request(256, 257),
+        }
+    )
+    output = schedule_output(
+        {"edge": 128, "after": 64, "decode": 1},
+        context_ids={"after"},
+        new_ids={"edge"},
+        preempted={"old"},
+    )
+    rec, path = recorder(tmp_path)
+    zero_prefix = dict.fromkeys(opprof._PREFIX_FIELDS, 0)
+    start = (100, 200, {"local": zero_prefix, "external": None})
+    monkeypatch.setattr(opprof.time, "monotonic_ns", lambda: 300)
+
+    rec.begin(sched, output, start)
+    assert rec.finalize(output, graph("FULL", 193, 256))
+    rec.close()
+
+    record = read_jsonl(path)[0]
+    assert record == {
+        "schema": 1,
+        "engine_id": "dp0-pid1",
+        "step_index": 0,
+        "submit_wall_ns": 100,
+        "submit_mono_ns": 200,
+        "model_executed": True,
+        "scheduled_requests": 3,
+        "decode_batch_size": 1,
+        "prefill_requests": 2,
+        "prefill_tokens": 192,
+        "decode_tokens": 1,
+        "chunked_prefill": {
+            "first": 0,
+            "middle": 0,
+            "final": 1,
+            "unsplit": 1,
+            "tokens": 192,
+            "chunk_size_hist": [0, 0, 1, 1, 0, 0, 0, 0, 0],
+        },
+        "context_length_hist": [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        "preemptions": 1,
+        "queues": {"running": 3, "waiting": 2, "deferred": 1},
+        "kv": {
+            "total_blocks": 100,
+            "free_blocks": 40,
+            "used_blocks": 60,
+            "usage": 0.6,
+        },
+        "prefix": {"local": zero_prefix, "external": None},
+        "complete_mono_ns": 300,
+        "cudagraph": {
+            "hit": True,
+            "runtime_mode": "FULL",
+            "unpadded_tokens": 193,
+            "bucket_tokens": 256,
+            "padding_tokens": 63,
+        },
+        "moe_expert_load": None,
+        "dropped_records_before": 0,
+    }
+
+
+@pytest.mark.parametrize(
+    ("was_chunk", "end", "target", "expected"),
+    [
+        (False, 64, 100, "first"),
+        (True, 80, 100, "middle"),
+        (True, 100, 100, "final"),
+        (False, 100, 100, "unsplit"),
+    ],
+)
+def test_chunk_classification(was_chunk, end, target, expected):
+    assert opprof.classify_chunk(was_chunk, end, target) == expected
+
+
+def test_async_pairing_out_of_order_and_double_finalize(tmp_path):
+    sched = scheduler({"a": request(10, 11), "b": request(200, 201)})
+    first = schedule_output({"a": 1})
+    second = schedule_output({"b": 1})
+    rec, path = recorder(tmp_path)
+    rec.begin(sched, first, rec.capture_start(sched))
+    rec.begin(sched, second, rec.capture_start(sched))
+    assert rec.finalize(second, graph())
+    assert rec.finalize(first, graph("NONE"))
+    with pytest.raises(AssertionError, match="already finalized"):
+        rec.finalize(second, graph())
+    assert not rec._pending
+    rec.close()
+    records = read_jsonl(path)[:-1]
+    assert [record["step_index"] for record in records] == [1, 0]
+    assert records[0]["context_length_hist"][1] == 1
+    assert records[1]["context_length_hist"][0] == 1
+
+
+def test_disabled_noop_and_log_stats_fail_fast(tmp_path):
+    assert opprof.OpProfRecorder.create("", dp_rank=0, log_stats=False) is None
+    with pytest.raises(ValueError, match="requires log stats"):
+        opprof.OpProfRecorder.create(str(tmp_path), dp_rank=0, log_stats=False)
+    assert not list(tmp_path.iterdir())
+
+
+def test_bounded_queue_drop_accounting(tmp_path):
+    sched = scheduler({str(i): request(i, i + 1) for i in range(3)})
+    rec, path = recorder(tmp_path, capacity=1, start=False)
+    assert emit(rec, sched, schedule_output({"0": 1}))
+    assert not emit(rec, sched, schedule_output({"1": 1}))
+    rec.writer.start()
+    rec.writer._queue.join()
+    assert emit(rec, sched, schedule_output({"2": 1}))
+    rec.close()
+
+    first, after_drop, footer = read_jsonl(path)
+    assert first["step_index"] == 0
+    assert after_drop["step_index"] == 2
+    assert after_drop["dropped_records_before"] == 1
+    assert footer["encoded_records"] == 3
+    assert footer["written_records"] == 2
+    assert footer["dropped_records"] == 1
+
+
+def test_writer_enospc_is_exposed_and_shutdown_is_bounded(
+    tmp_path, monkeypatch, caplog
+):
+    sched = scheduler(
+        {
+            "first": request(0, 1),
+            "after_failure": request(1, 2),
+        }
+    )
+    rec, _ = recorder(tmp_path, capacity=1, start=False)
+    assert emit(rec, sched, schedule_output({"first": 1}))
+
+    real_file = rec.writer._file
+
+    def fail_enospc(*_args, **_kwargs):
+        raise OSError(errno.ENOSPC, "No space left on device")
+
+    failing_file = SimpleNamespace(
+        write=fail_enospc,
+        flush=fail_enospc,
+        close=real_file.close,
+    )
+    monkeypatch.setattr(rec.writer, "_file", failing_file)
+    caplog.set_level(logging.ERROR, logger=opprof.__name__)
+
+    rec.writer.start()
+    rec.writer._thread.join(timeout=1.0)
+    assert not rec.writer._thread.is_alive()
+
+    producer_result = emit(
+        rec, sched, schedule_output({"after_failure": 1})
+    )
+
+    closer = threading.Thread(target=rec.close, daemon=True)
+    closer.start()
+    closer.join(timeout=1.0)
+    assert not closer.is_alive(), "OpProf close blocked after writer failure"
+    assert not producer_result
+    assert rec.writer.dropped_records == 1
+    assert rec.failed
+    assert isinstance(rec.failure, OSError)
+    assert rec.failure.errno == errno.ENOSPC
+    errors = [
+        record
+        for record in caplog.records
+        if "OpProf writer failed" in record.getMessage()
+    ]
+    assert len(errors) == 1
+
+
+def test_shutdown_flush_is_idempotent(tmp_path):
+    sched = scheduler({"decode": request(8, 9)})
+    rec, path = recorder(tmp_path)
+    assert emit(rec, sched, schedule_output({"decode": 1}))
+    rec.close()
+    rec.close()
+    record, footer = read_jsonl(path)
+    assert record["step_index"] == 0
+    assert footer["written_records"] == 1
+    assert path.stat().st_size > 0
+
+
+def test_piecewise_cudagraph_record_preserved(tmp_path):
+    sched = scheduler({"decode": request(4096, 4097)})
+    output = schedule_output({"decode": 1})
+    rec, path = recorder(tmp_path)
+    assert emit(rec, sched, output, graph("PIECEWISE", 513, 520))
+    rec.close()
+    record = read_jsonl(path)[0]
+    assert record["cudagraph"] == {
+        "hit": True,
+        "runtime_mode": "PIECEWISE",
+        "unpadded_tokens": 513,
+        "bucket_tokens": 520,
+        "padding_tokens": 7,
+    }
+
+
+def test_zero_scheduled_tokens_finalize_without_cudagraph(tmp_path):
+    sched = scheduler({})
+    output = schedule_output({})
+    rec, path = recorder(tmp_path)
+    rec.begin(sched, output, rec.capture_start(sched))
+    assert rec._pending
+
+    assert rec.finalize(output, cudagraph_stat=None)
+    assert not rec._pending
+    rec.close()
+
+    record = read_jsonl(path)[0]
+    assert record["model_executed"] is False
+    assert record["scheduled_requests"] == 0
+    assert record["prefill_requests"] == 0
+    assert record["prefill_tokens"] == 0
+    assert record["decode_batch_size"] == 0
+    assert record["decode_tokens"] == 0
+    assert record["context_length_hist"] == [0] * 12
+    assert record["chunked_prefill"] == {
+        "first": 0,
+        "middle": 0,
+        "final": 0,
+        "unsplit": 0,
+        "tokens": 0,
+        "chunk_size_hist": [0] * 9,
+    }
+    assert record["cudagraph"] == {
+        "hit": False,
+        "runtime_mode": "NONE",
+        "unpadded_tokens": 0,
+        "bucket_tokens": 0,
+        "padding_tokens": 0,
+    }
-- 
2.43.0

