gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Browse Source 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-07-13 11:06:10 +08:00
parent 607e88da3c
commit d5b276180d
412 changed files with 125056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
runs/opprof-phase6/phase6/cells/tp2_mns8/anchor-0.4921875.log Normal file
View File

@@ -0,0 +1 @@
{"cell": "tp2_mns8", "anchor": 0.4921875, "kind": "anchor", "pass_rate": 0.6914498141263941, "feasible": false}

74
runs/opprof-phase6/phase6/cells/tp2_mns8/anchor-0.4921875/result.json Normal file
View File

@@ -0,0 +1,74 @@
{
"anchor": 0.4921875,
"cell": "tp2_mns8",
"early_stop_reason": "slo_pass_rate_unrecoverable",
"early_stopped": true,
"exact_output_count": 216,
"feasible": false,
"interval": {
"elapsed_s": 51.56724132,
"end_mono_ns": 201214250324754,
"end_wall_ns": 1783869641376751823,
"start_mono_ns": 201162683083434,
"start_wall_ns": 1783869589809510519
},
"invariants": {
"arrival_nondecreasing": true,
"exact_output_or_failed": true,
"outcomes_cover_selected": true,
"raw_lengths_present": true,
"selected_nonempty": true,
"warmup_16": true,
"warmup_exact_16": true,
"warmup_long": true
},
"kind": "anchor",
"mns": 8,
"observed_count": 269,
"pass_rate": 0.6914498141263941,
"schema": 1,
"selection": {
"arrival_order_sha256": "ffbc7b8dd324c8e745654110a2412b03ddade891beb2bfe022ffde6502b0184b",
"arrival_s": {
"distinct_n": 269,
"finite_n": 269,
"max": 59.78950000000005,
"min": 0.008300000000008368,
"missing_n": 0,
"n": 269
},
"count": 269,
"long_gt4096": 95,
"offered_req_s": 4.483333333333333,
"offered_req_s_per_gpu": 2.2416666666666667,
"raw_input_tokens": {
"distinct_n": 250,
"finite_n": 269,
"max": 8149.0,
"min": 71.0,
"missing_n": 0,
"n": 269
},
"raw_length_order_sha256": "bc951350376a487d34d2def78fd32a2286e23bf8a7f481ff578009e12e3f1fb8",
"request_id_order_sha256": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c"
},
"slo_pass_count": 186,
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
"tp": 2,
"tpot_ms": {
"distinct_n": 216,
"finite_n": 216,
"max": 22.936211125952685,
"min": 5.354799519524008,
"missing_n": 53,
"n": 269
},
"ttft_ms": {
"distinct_n": 216,
"finite_n": 216,
"max": 3692.623541021021,
"min": 40.143018995877355,
"missing_n": 53,
"n": 269
}
}

1
runs/opprof-phase6/phase6/cells/tp2_mns8/anchor-0.49609375.log Normal file
View File

@@ -0,0 +1 @@
{"cell": "tp2_mns8", "anchor": 0.49609375, "kind": "anchor", "pass_rate": 0.09157509157509157, "feasible": false}

74
runs/opprof-phase6/phase6/cells/tp2_mns8/anchor-0.49609375/result.json Normal file
View File

@@ -0,0 +1,74 @@
{
"anchor": 0.49609375,
"cell": "tp2_mns8",
"early_stop_reason": "slo_pass_rate_unrecoverable",
"early_stopped": true,
"exact_output_count": 87,
"feasible": false,
"interval": {
"elapsed_s": 33.978401963,
"end_mono_ns": 201156432397399,
"end_wall_ns": 1783869583558824468,
"start_mono_ns": 201122453995436,
"start_wall_ns": 1783869549580422810
},
"invariants": {
"arrival_nondecreasing": true,
"exact_output_or_failed": true,
"outcomes_cover_selected": true,
"raw_lengths_present": true,
"selected_nonempty": true,
"warmup_16": true,
"warmup_exact_16": true,
"warmup_long": true
},
"kind": "anchor",
"mns": 8,
"observed_count": 273,
"pass_rate": 0.09157509157509157,
"schema": 1,
"selection": {
"arrival_order_sha256": "41168f4bd02f3ef4e83b7440a3dab7458f08df9c2bbd4452fad076b4a2e0eaed",
"arrival_s": {
"distinct_n": 273,
"finite_n": 273,
"max": 59.78950000000005,
"min": 0.008300000000008368,
"missing_n": 0,
"n": 273
},
"count": 273,
"long_gt4096": 97,
"offered_req_s": 4.55,
"offered_req_s_per_gpu": 2.275,
"raw_input_tokens": {
"distinct_n": 253,
"finite_n": 273,
"max": 8149.0,
"min": 71.0,
"missing_n": 0,
"n": 273
},
"raw_length_order_sha256": "9e38ed1766e3933dc05a7ced5d0a73fed47ce769800f0d2c10014af0ef823f28",
"request_id_order_sha256": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80"
},
"slo_pass_count": 25,
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
"tp": 2,
"tpot_ms": {
"distinct_n": 87,
"finite_n": 87,
"max": 38.90758789764116,
"min": 6.895991826913957,
"missing_n": 186,
"n": 273
},
"ttft_ms": {
"distinct_n": 87,
"finite_n": 87,
"max": 16124.392641999293,
"min": 52.06192599143833,
"missing_n": 186,
"n": 273
}
}

22
runs/opprof-phase6/phase6/cells/tp2_mns8/cell-valid.json Normal file
View File

@@ -0,0 +1,22 @@
{
"accounting_mode": "graceful-footer",
"cell": "tp2_mns8",
"invariants": {
"anchor_intervals_present": true,
"compile_capture_pre_ready": true,
"encoded_balanced": true,
"footer_sidecar_agrees": true,
"last_step_matches": true,
"layer1_contiguous": true,
"layer1_zero_drops": true,
"one_footer_last": true,
"one_ready_marker": true,
"sidecar_final": true,
"warmup_exact_16": true,
"warmup_long": true,
"written_matches_records": true
},
"layer1_records": 6687,
"post_ready_capture_events": [],
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/opprof/opprof-v1-dp0-pid2661256-1783869521184806299.jsonl"
}

4
runs/opprof-phase6/phase6/cells/tp2_mns8/commands.log Normal file
View File

@@ -0,0 +1,4 @@
SERVER taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8510 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 8 --tensor-parallel-size 2 --shutdown-timeout 120
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.49609375 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/warmup
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.49609375 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/anchor-0.49609375
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.4921875 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/anchor-0.4921875

1
runs/opprof-phase6/phase6/cells/tp2_mns8/opprof/opprof-v1-dp0-pid2661256-1783869521184806299.jsonl.footer.json Normal file
View File

@@ -0,0 +1 @@
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661256-1783869521184806299.jsonl","encoded_records":6687,"written_records":6687,"dropped_records":0,"last_step_index":6686,"checkpoint_wall_ns":1783869792959224277,"flush_interval_seconds":1.0,"final":true}

486
runs/opprof-phase6/phase6/cells/tp2_mns8/server.log Normal file
View File

@@ -0,0 +1,486 @@
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339]
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] █ █ █▄ ▄█
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339]
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8510, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'tensor_parallel_size': 2, 'max_num_batched_tokens': 8192, 'max_num_seqs': 8, 'shutdown_timeout': 120}
(APIServer pid=2660454) INFO 07-12 15:16:40 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
(APIServer pid=2660454) INFO 07-12 15:16:40 [model.py:1725] Using max model len 40960
(APIServer pid=2660454) INFO 07-12 15:16:40 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
(APIServer pid=2660454) INFO 07-12 15:16:40 [vllm.py:1006] Asynchronous scheduling is enabled.
(APIServer pid=2660454) INFO 07-12 15:16:40 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
(APIServer pid=2660454) INFO 07-12 15:16:45 [compilation.py:310] Enabled custom fusions: allreduce_rms
(EngineCore pid=2661256) INFO 07-12 15:16:55 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': True, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 16, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
(EngineCore pid=2661256) WARNING 07-12 15:16:55 [multiproc_executor.py:1063] Reducing Torch parallelism from 40 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
(EngineCore pid=2661256) INFO 07-12 15:16:55 [multiproc_executor.py:140] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.27.132.244 (local), world_size=2, local_world_size=2
(Worker pid=2661580) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:42171 backend=nccl
(Worker pid=2661581) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:42171 backend=nccl
(Worker pid=2661580) INFO 07-12 15:17:06 [pynccl.py:113] vLLM is using nccl==2.28.9
(Worker pid=2661580) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['CUSTOM', 'SYMM_MEM', 'PYNCCL'] all-reduce backends (in dispatch order) for group 'tp:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
(Worker pid=2661580) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['PYNCCL'] all-reduce backends (in dispatch order) for group 'ep:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
(Worker pid=2661580) INFO 07-12 15:17:09 [parallel_state.py:1923] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
(Worker pid=2661580) INFO 07-12 15:17:10 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [flash_attn.py:670] Using FlashAttention version 3
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1275.26 GiB.
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:09, 1.61it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:09, 1.53it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:01<00:08, 1.51it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:02<00:08, 1.47it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:03<00:07, 1.45it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:04<00:06, 1.43it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:04<00:06, 1.44it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:05<00:05, 1.45it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:06<00:04, 1.44it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:06<00:04, 1.44it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:07<00:03, 1.46it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:08<00:02, 1.46it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:08<00:02, 1.46it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:09<00:01, 1.46it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:10<00:00, 1.46it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.89it/s]
(Worker_TP0 pid=2661580)
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.53it/s]
(Worker_TP0 pid=2661580)
(Worker_TP0 pid=2661580) INFO 07-12 15:17:21 [default_loader.py:430] Loading weights took 10.45 seconds
(Worker_TP0 pid=2661580) INFO 07-12 15:17:21 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
(Worker_TP0 pid=2661580) INFO 07-12 15:17:22 [gpu_model_runner.py:5259] Model loading took 28.46 GiB memory and 10.837326 seconds
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/0c37cd99d7/rank_0_0/backbone for vLLM's torch.compile
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [backends.py:1148] Dynamo bytecode transform time: 9.74 s
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [flashinfer_all_reduce.py:112] Auto-selected flashinfer allreduce backend: trtllm
(Worker_TP0 pid=2661580) /tmp/wjh-opprof-phase2-dash0-20260711/.venv/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
(Worker_TP0 pid=2661580) return func(*args, **kwargs)
[rank0]:[W712 15:17:32.753967979 ProcessGroupNCCL.cpp:5188] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
(Worker_TP0 pid=2661580) INFO 07-12 15:17:33 [flashinfer_all_reduce.py:152] Initialized FlashInfer Allreduce norm fusion workspace with backend=trtllm
(Worker_TP0 pid=2661580) INFO 07-12 15:17:56 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
(Worker_TP0 pid=2661580) INFO 07-12 15:18:03 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 30.05 s
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/7db77e4784ad115c73ccf5b7472a39190ea4c85a91f415675a26d40f2c108d60/rank_0_0/model
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [monitor.py:53] torch.compile took 56.36 s in total
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20.json for MoE layer.
(Worker_TP0 pid=2661580) INFO 07-12 15:18:21 [monitor.py:81] Initial profiling/warmup run took 2.52 s
(EngineCore pid=2661256) INFO 07-12 15:18:23 [shm_broadcast.py:705] No available shared memory broadcast block found in 60 seconds. This typically happens when some processes are hanging or doing some time-consuming work (e.g. compilation, weight/kv cache quantization).
(Worker_TP0 pid=2661580) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
(Worker_TP1 pid=2661581) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
(Worker_TP0 pid=2661580) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
(Worker_TP1 pid=2661581) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
(Worker_TP0 pid=2661580) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
(Worker_TP1 pid=2661581) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [gpu_worker.py:508] Available KV cache memory: 57.37 GiB
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
(Worker_TP1 pid=2661581) INFO 07-12 15:18:35 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
(EngineCore pid=2661256) INFO 07-12 15:18:35 [kv_cache_utils.py:2146] GPU KV cache size: 1,253,168 tokens
(EngineCore pid=2661256) INFO 07-12 15:18:35 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 30.59x
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
(Worker_TP0 pid=2661580) 2026-07-12 15:18:35,188 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
(Worker_TP1 pid=2661581) 2026-07-12 15:18:35,200 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
(Worker_TP0 pid=2661580) 2026-07-12 15:18:35,244 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
(Worker_TP1 pid=2661581) 2026-07-12 15:18:35,270 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
(Worker_TP0 pid=2661580)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/5 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 20%|██ | 1/5 [00:00<00:00, 8.16it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 40%|████ | 2/5 [00:00<00:00, 8.50it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 60%|██████ | 3/5 [00:00<00:00, 8.72it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 4/5 [00:01<00:00, 1.89it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 1.37it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 1.93it/s]
(Worker_TP0 pid=2661580)
Capturing CUDA graphs (decode, FULL): 0%| | 0/4 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 25%|██▌ | 1/4 [00:00<00:00, 8.58it/s]
Capturing CUDA graphs (decode, FULL): 50%|█████ | 2/4 [00:00<00:00, 9.08it/s]
Capturing CUDA graphs (decode, FULL): 75%|███████▌ | 3/4 [00:00<00:00, 9.18it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 9.44it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 9.28it/s]
(Worker_TP0 pid=2661580) INFO 07-12 15:18:38 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
(Worker_TP1 pid=2661581) INFO 07-12 15:18:38 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
(Worker_TP1 pid=2661581) INFO 07-12 15:18:39 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (14.3%).
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [gpu_model_runner.py:6660] Graph capturing finished in 4 secs, took 0.08 GiB
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (14.3%).
(Worker_TP1 pid=2661581) INFO 07-12 15:18:39 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
(EngineCore pid=2661256) INFO 07-12 15:18:39 [core.py:337] init engine (profile, create kv cache, warmup model) took 77.56 s (compilation: 56.36 s)
(EngineCore pid=2661256) INFO 07-12 15:18:41 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/opprof/opprof-v1-dp0-pid2661256-1783869521184806299.jsonl
(EngineCore pid=2661256) INFO 07-12 15:18:41 [vllm.py:1006] Asynchronous scheduling is enabled.
(EngineCore pid=2661256) INFO 07-12 15:18:41 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
(EngineCore pid=2661256) INFO 07-12 15:18:42 [compilation.py:310] Enabled custom fusions: allreduce_rms
(APIServer pid=2660454) INFO 07-12 15:18:42 [api_server.py:577] Supported tasks: ['generate']
(APIServer pid=2660454) WARNING 07-12 15:18:42 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
(APIServer pid=2660454) INFO 07-12 15:18:42 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
(APIServer pid=2660454) INFO 07-12 15:18:43 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8510
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:37] Available routes are:
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /docs, Methods: GET, HEAD
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /load, Methods: GET
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /version, Methods: GET
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /health, Methods: GET
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /metrics, Methods: GET
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /tokenize, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /detokenize, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/models, Methods: GET
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /ping, Methods: GET
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /ping, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /invocations, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/messages, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /generative_scoring, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions/render, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
(APIServer pid=2660454) INFO: Started server process [2660454]
(APIServer pid=2660454) INFO: Waiting for application startup.
(APIServer pid=2660454) INFO: Application startup complete.
(APIServer pid=2660454) INFO: 127.0.0.1:34364 - "GET /v1/models HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(Worker_TP0 pid=2661580) WARNING 07-12 15:18:54 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
(APIServer pid=2660454) INFO: 127.0.0.1:43026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:43128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(Worker_TP0 pid=2661580) WARNING 07-12 15:18:57 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
(APIServer pid=2660454) INFO: 127.0.0.1:43140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:19:03 [loggers.py:273] Engine 000: Avg prompt throughput: 5201.2 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.4%
(APIServer pid=2660454) INFO: 127.0.0.1:52192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:19:13 [loggers.py:273] Engine 000: Avg prompt throughput: 11.8 tokens/s, Avg generation throughput: 110.7 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 50.0%
(APIServer pid=2660454) INFO: 127.0.0.1:41958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:41986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42094 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42190 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42296 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:19:23 [loggers.py:273] Engine 000: Avg prompt throughput: 5066.8 tokens/s, Avg generation throughput: 298.5 tokens/s, Running: 8 reqs, Waiting: 33 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 36.8%
(APIServer pid=2660454) INFO: 127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42368 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:42452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:19:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5845.7 tokens/s, Avg generation throughput: 315.7 tokens/s, Running: 7 reqs, Waiting: 26 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 29.2%
(APIServer pid=2660454) INFO 07-12 15:19:43 [loggers.py:273] Engine 000: Avg prompt throughput: 6481.6 tokens/s, Avg generation throughput: 371.0 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 24.8%
(APIServer pid=2660454) INFO: 127.0.0.1:35332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:47932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:47944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:47946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:47956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:47972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:47986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:47988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:47996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:19:53 [loggers.py:273] Engine 000: Avg prompt throughput: 16.3 tokens/s, Avg generation throughput: 193.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 35.5%
(APIServer pid=2660454) INFO: 127.0.0.1:48022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48154 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48172 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:48332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35846 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:20:03 [loggers.py:273] Engine 000: Avg prompt throughput: 45.0 tokens/s, Avg generation throughput: 646.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 54.1%
(APIServer pid=2660454) INFO: 127.0.0.1:35888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:35988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:36130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:20:13 [loggers.py:273] Engine 000: Avg prompt throughput: 6622.6 tokens/s, Avg generation throughput: 499.2 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 52.5%
(APIServer pid=2660454) INFO: 127.0.0.1:52132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:52482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:20:23 [loggers.py:273] Engine 000: Avg prompt throughput: 10999.7 tokens/s, Avg generation throughput: 475.5 tokens/s, Running: 8 reqs, Waiting: 8 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 45.6%
(APIServer pid=2660454) INFO: 127.0.0.1:57836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57844 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57864 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57892 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:57994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:58058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:32944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:32948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:32954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:32962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:32972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:32976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:32978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:32990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO 07-12 15:20:33 [loggers.py:273] Engine 000: Avg prompt throughput: 10684.1 tokens/s, Avg generation throughput: 487.2 tokens/s, Running: 8 reqs, Waiting: 14 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 40.8%
(APIServer pid=2660454) INFO: 127.0.0.1:33078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33134 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
(APIServer pid=2660454) INFO: 127.0.0.1:33158 - "POST /v1/chat/completions HTTP/1.1" 200 OK

1
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup.log Normal file
View File

@@ -0,0 +1 @@
{"cell": "tp2_mns8", "anchor": 0.49609375, "kind": "warmup", "pass_rate": 0.625, "feasible": false}

74
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup/result.json Normal file
View File

@@ -0,0 +1,74 @@
{
"anchor": 0.49609375,
"cell": "tp2_mns8",
"early_stop_reason": "",
"early_stopped": false,
"exact_output_count": 16,
"feasible": false,
"interval": {
"elapsed_s": 9.026111125,
"end_mono_ns": 201115775651265,
"end_wall_ns": 1783869542902078194,
"start_mono_ns": 201106749540140,
"start_wall_ns": 1783869533875967269
},
"invariants": {
"arrival_nondecreasing": true,
"exact_output_or_failed": true,
"outcomes_cover_selected": true,
"raw_lengths_present": true,
"selected_nonempty": true,
"warmup_16": true,
"warmup_exact_16": true,
"warmup_long": true
},
"kind": "warmup",
"mns": 8,
"observed_count": 16,
"pass_rate": 0.625,
"schema": 1,
"selection": {
"arrival_order_sha256": "818d316f68acc6502ed33a877bf88c2ba9b57a1ff6d6780621db0d30eddf3494",
"arrival_s": {
"distinct_n": 16,
"finite_n": 16,
"max": 3.4938000000000105,
"min": 0.0,
"missing_n": 0,
"n": 16
},
"count": 16,
"long_gt4096": 7,
"offered_req_s": 0.26666666666666666,
"offered_req_s_per_gpu": 0.13333333333333333,
"raw_input_tokens": {
"distinct_n": 16,
"finite_n": 16,
"max": 7270.0,
"min": 71.0,
"missing_n": 0,
"n": 16
},
"raw_length_order_sha256": "2eb3b78b31efc9354c64c8caa95f08353330553d1ab745a8e0ceab038a3a3d5e",
"request_id_order_sha256": "0009512d07ec4c5bb2dd1283c767967a6dfd5bc4c7b7d65fb26169d39a98832e"
},
"slo_pass_count": 10,
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
"tp": 2,
"tpot_ms": {
"distinct_n": 16,
"finite_n": 16,
"max": 39.46136329144306,
"min": 7.347993740140952,
"missing_n": 0,
"n": 16
},
"ttft_ms": {
"distinct_n": 16,
"finite_n": 16,
"max": 4594.941660994664,
"min": 762.8166690119542,
"missing_n": 0,
"n": 16
}
}