gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Browse Source 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-07-13 11:06:10 +08:00
parent 607e88da3c
commit d5b276180d
412 changed files with 125056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.24609375.log Normal file
View File

@@ -0,0 +1 @@
{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

74
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.24609375/result.json Normal file
View File

@@ -0,0 +1,74 @@
{
"anchor": 0.24609375,
"cell": "tp1_mns16",
"early_stop_reason": "",
"early_stopped": false,
"exact_output_count": 141,
"feasible": true,
"interval": {
"elapsed_s": 61.285616071,
"end_mono_ns": 208628026400377,
"end_wall_ns": 1783877055152827630,
"start_mono_ns": 208566740784306,
"start_wall_ns": 1783876993867211598
},
"invariants": {
"arrival_nondecreasing": true,
"exact_output_or_failed": true,
"outcomes_cover_selected": true,
"raw_lengths_present": true,
"selected_nonempty": true,
"warmup_16": true,
"warmup_exact_16": true,
"warmup_long": true
},
"kind": "anchor",
"mns": 16,
"observed_count": 141,
"pass_rate": 1.0,
"schema": 1,
"selection": {
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
"arrival_s": {
"distinct_n": 141,
"finite_n": 141,
"max": 59.78950000000005,
"min": 0.008300000000008368,
"missing_n": 0,
"n": 141
},
"count": 141,
"long_gt4096": 50,
"offered_req_s": 2.35,
"offered_req_s_per_gpu": 2.35,
"raw_input_tokens": {
"distinct_n": 133,
"finite_n": 141,
"max": 8149.0,
"min": 72.0,
"missing_n": 0,
"n": 141
},
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
},
"slo_pass_count": 141,
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
"tp": 1,
"tpot_ms": {
"distinct_n": 141,
"finite_n": 141,
"max": 39.26863328358488,
"min": 4.784946039540869,
"missing_n": 0,
"n": 141
},
"ttft_ms": {
"distinct_n": 141,
"finite_n": 141,
"max": 1786.641047016019,
"min": 38.60557498410344,
"missing_n": 0,
"n": 141
}
}

1
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.25.log Normal file
View File

@@ -0,0 +1 @@
{"cell": "tp1_mns16", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

74
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.25/result.json Normal file
View File

@@ -0,0 +1,74 @@
{
"anchor": 0.25,
"cell": "tp1_mns16",
"early_stop_reason": "",
"early_stopped": false,
"exact_output_count": 143,
"feasible": true,
"interval": {
"elapsed_s": 61.484334594,
"end_mono_ns": 208696038541120,
"end_wall_ns": 1783877123164968015,
"start_mono_ns": 208634554206526,
"start_wall_ns": 1783877061680633802
},
"invariants": {
"arrival_nondecreasing": true,
"exact_output_or_failed": true,
"outcomes_cover_selected": true,
"raw_lengths_present": true,
"selected_nonempty": true,
"warmup_16": true,
"warmup_exact_16": true,
"warmup_long": true
},
"kind": "anchor",
"mns": 16,
"observed_count": 143,
"pass_rate": 1.0,
"schema": 1,
"selection": {
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
"arrival_s": {
"distinct_n": 143,
"finite_n": 143,
"max": 59.78950000000005,
"min": 0.008300000000008368,
"missing_n": 0,
"n": 143
},
"count": 143,
"long_gt4096": 51,
"offered_req_s": 2.3833333333333333,
"offered_req_s_per_gpu": 2.3833333333333333,
"raw_input_tokens": {
"distinct_n": 135,
"finite_n": 143,
"max": 8149.0,
"min": 72.0,
"missing_n": 0,
"n": 143
},
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
},
"slo_pass_count": 143,
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
"tp": 1,
"tpot_ms": {
"distinct_n": 143,
"finite_n": 143,
"max": 44.45809762976243,
"min": 4.488573606239676,
"missing_n": 0,
"n": 143
},
"ttft_ms": {
"distinct_n": 143,
"finite_n": 143,
"max": 1778.0322160106152,
"min": 47.90777899324894,
"missing_n": 0,
"n": 143
}
}

1
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.5.log Normal file
View File

@@ -0,0 +1 @@
{"cell": "tp1_mns16", "anchor": 0.5, "kind": "anchor", "pass_rate": 0.15579710144927536, "feasible": false}

74
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.5/result.json Normal file
View File

@@ -0,0 +1,74 @@
{
"anchor": 0.5,
"cell": "tp1_mns16",
"early_stop_reason": "slo_pass_rate_unrecoverable",
"early_stopped": true,
"exact_output_count": 92,
"feasible": false,
"interval": {
"elapsed_s": 26.818138737,
"end_mono_ns": 208796704305524,
"end_wall_ns": 1783877223830732816,
"start_mono_ns": 208769886166787,
"start_wall_ns": 1783877197012593929
},
"invariants": {
"arrival_nondecreasing": true,
"exact_output_or_failed": true,
"outcomes_cover_selected": true,
"raw_lengths_present": true,
"selected_nonempty": true,
"warmup_16": true,
"warmup_exact_16": true,
"warmup_long": true
},
"kind": "anchor",
"mns": 16,
"observed_count": 276,
"pass_rate": 0.15579710144927536,
"schema": 1,
"selection": {
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
"arrival_s": {
"distinct_n": 276,
"finite_n": 276,
"max": 59.78950000000005,
"min": 0.008300000000008368,
"missing_n": 0,
"n": 276
},
"count": 276,
"long_gt4096": 99,
"offered_req_s": 4.6,
"offered_req_s_per_gpu": 4.6,
"raw_input_tokens": {
"distinct_n": 256,
"finite_n": 276,
"max": 8149.0,
"min": 71.0,
"missing_n": 0,
"n": 276
},
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
},
"slo_pass_count": 43,
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
"tp": 1,
"tpot_ms": {
"distinct_n": 92,
"finite_n": 92,
"max": 40.052486527445325,
"min": 11.32848871666498,
"missing_n": 184,
"n": 276
},
"ttft_ms": {
"distinct_n": 92,
"finite_n": 92,
"max": 6538.087566004833,
"min": 66.57706701662391,
"missing_n": 184,
"n": 276
}
}

22
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/cell-valid.json Normal file
View File

@@ -0,0 +1,22 @@
{
"accounting_mode": "graceful-footer",
"cell": "tp1_mns16",
"invariants": {
"anchor_intervals_present": true,
"compile_capture_pre_ready": true,
"encoded_balanced": true,
"footer_sidecar_agrees": true,
"last_step_matches": true,
"layer1_contiguous": true,
"layer1_zero_drops": true,
"one_footer_last": true,
"one_ready_marker": true,
"sidecar_final": true,
"warmup_exact_16": true,
"warmup_long": true,
"written_matches_records": true
},
"layer1_records": 14174,
"post_ready_capture_events": [],
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2781473-1783876972093434233.jsonl"
}

6
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/commands.log Normal file
View File

@@ -0,0 +1,6 @@
SERVER taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8709 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 1 --shutdown-timeout 120
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/warmup
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.24609375
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.25 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.25
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.25 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/confirm-1-anchor-0.25
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.5 --tp 1 --mns 16 --base-url http://127.0.0.1:8709 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/anchor-0.5

1
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/confirm-1-anchor-0.25.log Normal file
View File

@@ -0,0 +1 @@
{"cell": "tp1_mns16", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}

74
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/confirm-1-anchor-0.25/result.json Normal file
View File

@@ -0,0 +1,74 @@
{
"anchor": 0.25,
"cell": "tp1_mns16",
"early_stop_reason": "",
"early_stopped": false,
"exact_output_count": 143,
"feasible": true,
"interval": {
"elapsed_s": 61.468118415,
"end_mono_ns": 208763972089008,
"end_wall_ns": 1783877191098516077,
"start_mono_ns": 208702503970593,
"start_wall_ns": 1783877129630397769
},
"invariants": {
"arrival_nondecreasing": true,
"exact_output_or_failed": true,
"outcomes_cover_selected": true,
"raw_lengths_present": true,
"selected_nonempty": true,
"warmup_16": true,
"warmup_exact_16": true,
"warmup_long": true
},
"kind": "anchor",
"mns": 16,
"observed_count": 143,
"pass_rate": 1.0,
"schema": 1,
"selection": {
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
"arrival_s": {
"distinct_n": 143,
"finite_n": 143,
"max": 59.78950000000005,
"min": 0.008300000000008368,
"missing_n": 0,
"n": 143
},
"count": 143,
"long_gt4096": 51,
"offered_req_s": 2.3833333333333333,
"offered_req_s_per_gpu": 2.3833333333333333,
"raw_input_tokens": {
"distinct_n": 135,
"finite_n": 143,
"max": 8149.0,
"min": 72.0,
"missing_n": 0,
"n": 143
},
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
},
"slo_pass_count": 143,
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
"tp": 1,
"tpot_ms": {
"distinct_n": 143,
"finite_n": 143,
"max": 44.40597959039696,
"min": 4.490311023564716,
"missing_n": 0,
"n": 143
},
"ttft_ms": {
"distinct_n": 143,
"finite_n": 143,
"max": 1776.1785559996497,
"min": 49.54166599782184,
"missing_n": 0,
"n": 143
}
}

1
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2781473-1783876972093434233.jsonl.footer.json Normal file
View File

@@ -0,0 +1 @@
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2781473-1783876972093434233.jsonl","encoded_records":14174,"written_records":14174,"dropped_records":0,"last_step_index":14173,"checkpoint_wall_ns":1783877224563096809,"flush_interval_seconds":1.0,"final":true}

686
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/server.log Normal file
View File

@@ -0,0 +1,686 @@
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339]
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339] █ █ █▄ ▄█
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:339]
(APIServer pid=2781293) INFO 07-12 17:22:11 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8709, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 16, 'shutdown_timeout': 120}
(APIServer pid=2781293) INFO 07-12 17:22:11 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
(APIServer pid=2781293) INFO 07-12 17:22:11 [model.py:1725] Using max model len 40960
(APIServer pid=2781293) INFO 07-12 17:22:11 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
(APIServer pid=2781293) INFO 07-12 17:22:11 [vllm.py:1006] Asynchronous scheduling is enabled.
(APIServer pid=2781293) INFO 07-12 17:22:11 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
(EngineCore pid=2781473) INFO 07-12 17:22:21 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 32, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
(EngineCore pid=2781473) INFO 07-12 17:22:23 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:35361 backend=nccl
(EngineCore pid=2781473) INFO 07-12 17:22:23 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
(EngineCore pid=2781473) INFO 07-12 17:22:24 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
(EngineCore pid=2781473) INFO 07-12 17:22:24 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
(EngineCore pid=2781473) INFO 07-12 17:22:25 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
(EngineCore pid=2781473) INFO 07-12 17:22:25 [flash_attn.py:670] Using FlashAttention version 3
(EngineCore pid=2781473) INFO 07-12 17:22:25 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
(EngineCore pid=2781473) INFO 07-12 17:22:25 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1288.56 GiB.
(EngineCore pid=2781473) INFO 07-12 17:22:25 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:12, 1.16it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:12, 1.13it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:02<00:11, 1.13it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:03<00:10, 1.11it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:04<00:09, 1.10it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:05<00:09, 1.09it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:06<00:08, 1.09it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:07<00:07, 1.10it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:08<00:06, 1.11it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:09<00:05, 1.11it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:09<00:04, 1.12it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:10<00:03, 1.12it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:11<00:02, 1.12it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:12<00:01, 1.11it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:13<00:00, 1.11it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:13<00:00, 1.42it/s]
(EngineCore pid=2781473)
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:13<00:00, 1.16it/s]
(EngineCore pid=2781473)
(EngineCore pid=2781473) INFO 07-12 17:22:39 [default_loader.py:430] Loading weights took 13.77 seconds
(EngineCore pid=2781473) INFO 07-12 17:22:39 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
(EngineCore pid=2781473) INFO 07-12 17:22:40 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 14.432905 seconds
(EngineCore pid=2781473) INFO 07-12 17:22:43 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/f4a50989f8/rank_0_0/backbone for vLLM's torch.compile
(EngineCore pid=2781473) INFO 07-12 17:22:43 [backends.py:1148] Dynamo bytecode transform time: 3.54 s
(EngineCore pid=2781473) INFO 07-12 17:22:46 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.334 s
(EngineCore pid=2781473) INFO 07-12 17:22:46 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/fe5a82dbe929018b7aea7dee05b5cd31a21fe2682aca78ee4cbf2b37c8a086d6/rank_0_0/model
(EngineCore pid=2781473) INFO 07-12 17:22:46 [monitor.py:53] torch.compile took 6.27 s in total
(EngineCore pid=2781473) INFO 07-12 17:22:46 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
(EngineCore pid=2781473) INFO 07-12 17:22:46 [monitor.py:81] Initial profiling/warmup run took 0.24 s
(EngineCore pid=2781473) INFO 07-12 17:22:47 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
(EngineCore pid=2781473) INFO 07-12 17:22:49 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
(EngineCore pid=2781473) INFO 07-12 17:22:49 [gpu_worker.py:508] Available KV cache memory: 29.43 GiB
(EngineCore pid=2781473) INFO 07-12 17:22:49 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
(EngineCore pid=2781473) INFO 07-12 17:22:49 [kv_cache_utils.py:2146] GPU KV cache size: 321,456 tokens
(EngineCore pid=2781473) INFO 07-12 17:22:49 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.85x
(EngineCore pid=2781473) INFO 07-12 17:22:49 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
(EngineCore pid=2781473) INFO 07-12 17:22:49 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
(EngineCore pid=2781473) 2026-07-12 17:22:49,532 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
(EngineCore pid=2781473) 2026-07-12 17:22:49,575 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
(EngineCore pid=2781473)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/7 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 29%|██▊ | 2/7 [00:00<00:00, 11.65it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 4/7 [00:00<00:00, 11.68it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▌ | 6/7 [00:00<00:00, 9.25it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00, 8.62it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00, 9.30it/s]
(EngineCore pid=2781473)
Capturing CUDA graphs (decode, FULL): 0%| | 0/5 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 40%|████ | 2/5 [00:00<00:00, 12.61it/s]
Capturing CUDA graphs (decode, FULL): 80%|████████ | 4/5 [00:00<00:00, 12.94it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 12.99it/s]
(EngineCore pid=2781473) INFO 07-12 17:22:51 [gpu_model_runner.py:6660] Graph capturing finished in 2 secs, took 0.10 GiB
(EngineCore pid=2781473) INFO 07-12 17:22:51 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.02 GiB (15.7%).
(EngineCore pid=2781473) INFO 07-12 17:22:51 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
(EngineCore pid=2781473) INFO 07-12 17:22:51 [core.py:337] init engine (profile, create kv cache, warmup model) took 11.53 s (compilation: 6.27 s)
(EngineCore pid=2781473) INFO 07-12 17:22:52 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/solo-authoritative/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2781473-1783876972093434233.jsonl
(EngineCore pid=2781473) INFO 07-12 17:22:52 [vllm.py:1006] Asynchronous scheduling is enabled.
(EngineCore pid=2781473) INFO 07-12 17:22:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
(APIServer pid=2781293) INFO 07-12 17:22:52 [api_server.py:577] Supported tasks: ['generate']
(APIServer pid=2781293) WARNING 07-12 17:22:52 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
(APIServer pid=2781293) INFO 07-12 17:22:52 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
(APIServer pid=2781293) INFO 07-12 17:22:52 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8709
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:37] Available routes are:
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /docs, Methods: HEAD, GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /load, Methods: GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /version, Methods: GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /health, Methods: GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /metrics, Methods: GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /tokenize, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /detokenize, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/models, Methods: GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /ping, Methods: GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /ping, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /invocations, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/responses, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/completions, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/messages, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /generative_scoring, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/completions/render, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
(APIServer pid=2781293) INFO 07-12 17:22:52 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
(APIServer pid=2781293) INFO: Started server process [2781293]
(APIServer pid=2781293) INFO: Waiting for application startup.
(APIServer pid=2781293) INFO: Application startup complete.
(APIServer pid=2781293) INFO: 127.0.0.1:48878 - "GET /v1/models HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(EngineCore pid=2781473) WARNING 07-12 17:22:59 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
(APIServer pid=2781293) INFO: 127.0.0.1:48896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(EngineCore pid=2781473) WARNING 07-12 17:22:59 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
(APIServer pid=2781293) INFO: 127.0.0.1:48916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42656 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:23:03 [loggers.py:273] Engine 000: Avg prompt throughput: 3723.9 tokens/s, Avg generation throughput: 84.3 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.0%, Prefix cache hit rate: 3.1%
(APIServer pid=2781293) INFO: 127.0.0.1:42688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:42728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:23:13 [loggers.py:273] Engine 000: Avg prompt throughput: 1237.3 tokens/s, Avg generation throughput: 112.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
(APIServer pid=2781293) INFO: 127.0.0.1:44654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:44796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:23:23 [loggers.py:273] Engine 000: Avg prompt throughput: 741.6 tokens/s, Avg generation throughput: 264.7 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 48.9%
(APIServer pid=2781293) INFO: 127.0.0.1:58852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:23:33 [loggers.py:273] Engine 000: Avg prompt throughput: 6168.6 tokens/s, Avg generation throughput: 349.2 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 33.8%
(APIServer pid=2781293) INFO: 127.0.0.1:48438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:48500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:23:43 [loggers.py:273] Engine 000: Avg prompt throughput: 2986.6 tokens/s, Avg generation throughput: 154.2 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 31.8%
(APIServer pid=2781293) INFO: 127.0.0.1:51202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51238 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:51334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:23:53 [loggers.py:273] Engine 000: Avg prompt throughput: 6698.0 tokens/s, Avg generation throughput: 274.8 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.2%, Prefix cache hit rate: 25.6%
(APIServer pid=2781293) INFO: 127.0.0.1:50738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:50920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:24:03 [loggers.py:273] Engine 000: Avg prompt throughput: 8748.6 tokens/s, Avg generation throughput: 383.1 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.6%, Prefix cache hit rate: 22.2%
(APIServer pid=2781293) INFO: 127.0.0.1:55266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:39550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:39554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:39556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:39568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:39576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:39590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:24:13 [loggers.py:273] Engine 000: Avg prompt throughput: 7763.4 tokens/s, Avg generation throughput: 314.2 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 19.4%
(APIServer pid=2781293) INFO: 127.0.0.1:39600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:39616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:39632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:24:23 [loggers.py:273] Engine 000: Avg prompt throughput: 2772.7 tokens/s, Avg generation throughput: 66.1 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.2%, Prefix cache hit rate: 18.4%
(APIServer pid=2781293) INFO: 127.0.0.1:55460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55528 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55572 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55574 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:55584 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:24:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5746.0 tokens/s, Avg generation throughput: 325.0 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 17.4%
(APIServer pid=2781293) INFO: 127.0.0.1:37882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:38066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36904 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:24:43 [loggers.py:273] Engine 000: Avg prompt throughput: 5414.5 tokens/s, Avg generation throughput: 327.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 16.9%
(APIServer pid=2781293) INFO: 127.0.0.1:36906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:36984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:24:53 [loggers.py:273] Engine 000: Avg prompt throughput: 4879.3 tokens/s, Avg generation throughput: 147.4 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.5%, Prefix cache hit rate: 16.4%
(APIServer pid=2781293) INFO: 127.0.0.1:32782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32854 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32904 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:32932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54162 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:25:03 [loggers.py:273] Engine 000: Avg prompt throughput: 6254.7 tokens/s, Avg generation throughput: 284.3 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.7%, Prefix cache hit rate: 15.6%
(APIServer pid=2781293) INFO: 127.0.0.1:54184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:54354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53694 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:25:13 [loggers.py:273] Engine 000: Avg prompt throughput: 8412.0 tokens/s, Avg generation throughput: 386.4 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.1%, Prefix cache hit rate: 14.9%
(APIServer pid=2781293) INFO: 127.0.0.1:53744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:53884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57844 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:25:23 [loggers.py:273] Engine 000: Avg prompt throughput: 7810.9 tokens/s, Avg generation throughput: 352.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 14.3%
(APIServer pid=2781293) INFO: 127.0.0.1:57866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:57882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60480 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:25:33 [loggers.py:273] Engine 000: Avg prompt throughput: 2806.5 tokens/s, Avg generation throughput: 102.5 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.6%, Prefix cache hit rate: 14.0%
(APIServer pid=2781293) INFO: 127.0.0.1:60504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60602 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60648 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:25:43 [loggers.py:273] Engine 000: Avg prompt throughput: 6210.2 tokens/s, Avg generation throughput: 300.2 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 13.7%
(APIServer pid=2781293) INFO: 127.0.0.1:43154 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:43302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:25:53 [loggers.py:273] Engine 000: Avg prompt throughput: 4335.6 tokens/s, Avg generation throughput: 281.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 13.6%
(APIServer pid=2781293) INFO: 127.0.0.1:33488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33538 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:33616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:26:03 [loggers.py:273] Engine 000: Avg prompt throughput: 4969.8 tokens/s, Avg generation throughput: 215.2 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 13.5%
(APIServer pid=2781293) INFO: 127.0.0.1:56072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56134 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56152 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56206 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:56240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:26:13 [loggers.py:273] Engine 000: Avg prompt throughput: 8656.3 tokens/s, Avg generation throughput: 225.0 tokens/s, Running: 15 reqs, Waiting: 1 reqs, GPU KV cache usage: 16.7%, Prefix cache hit rate: 13.1%
(APIServer pid=2781293) INFO: 127.0.0.1:40818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:40990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:41000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:41016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:26:23 [loggers.py:273] Engine 000: Avg prompt throughput: 7267.6 tokens/s, Avg generation throughput: 393.8 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.3%, Prefix cache hit rate: 12.8%
(APIServer pid=2781293) INFO: 127.0.0.1:49182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49296 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49302 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:49326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:26:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5850.9 tokens/s, Avg generation throughput: 318.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 12.6%
(APIServer pid=2781293) INFO: 127.0.0.1:58378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:58518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:26:43 [loggers.py:273] Engine 000: Avg prompt throughput: 6442.0 tokens/s, Avg generation throughput: 165.8 tokens/s, Running: 15 reqs, Waiting: 2 reqs, GPU KV cache usage: 15.1%, Prefix cache hit rate: 12.2%
(APIServer pid=2781293) INFO: 127.0.0.1:59972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:59998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60104 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:60332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37676 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO 07-12 17:26:53 [loggers.py:273] Engine 000: Avg prompt throughput: 9543.6 tokens/s, Avg generation throughput: 465.2 tokens/s, Running: 15 reqs, Waiting: 19 reqs, GPU KV cache usage: 13.3%, Prefix cache hit rate: 12.1%
(APIServer pid=2781293) INFO: 127.0.0.1:37764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2781293) INFO: 127.0.0.1:37780 - "POST /v1/chat/completions HTTP/1.1" 200 OK

1
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/warmup.log Normal file
View File

@@ -0,0 +1 @@
{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "warmup", "pass_rate": 1.0, "feasible": true}

74
runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/warmup/result.json Normal file
View File

@@ -0,0 +1,74 @@
{
"anchor": 0.24609375,
"cell": "tp1_mns16",
"early_stop_reason": "",
"early_stopped": false,
"exact_output_count": 16,
"feasible": true,
"interval": {
"elapsed_s": 8.0858641,
"end_mono_ns": 208560052503917,
"end_wall_ns": 1783876987178930964,
"start_mono_ns": 208551966639817,
"start_wall_ns": 1783876979093067067
},
"invariants": {
"arrival_nondecreasing": true,
"exact_output_or_failed": true,
"outcomes_cover_selected": true,
"raw_lengths_present": true,
"selected_nonempty": true,
"warmup_16": true,
"warmup_exact_16": true,
"warmup_long": true
},
"kind": "warmup",
"mns": 16,
"observed_count": 16,
"pass_rate": 1.0,
"schema": 1,
"selection": {
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
"arrival_s": {
"distinct_n": 16,
"finite_n": 16,
"max": 6.901699999999983,
"min": 0.0,
"missing_n": 0,
"n": 16
},
"count": 16,
"long_gt4096": 8,
"offered_req_s": 0.26666666666666666,
"offered_req_s_per_gpu": 0.26666666666666666,
"raw_input_tokens": {
"distinct_n": 16,
"finite_n": 16,
"max": 7270.0,
"min": 72.0,
"missing_n": 0,
"n": 16
},
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
},
"slo_pass_count": 16,
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
"tp": 1,
"tpot_ms": {
"distinct_n": 16,
"finite_n": 16,
"max": 28.565764511746583,
"min": 5.336474936856983,
"missing_n": 0,
"n": 16
},
"ttft_ms": {
"distinct_n": 16,
"finite_n": 16,
"max": 692.4971300177276,
"min": 73.91095400089398,
"missing_n": 0,
"n": 16
}
}