Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)
Workload-conditioned operator profiling on patched vLLM 0.24.0 + Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64 raggedness, 8-45% token-efficiency loss vs rectangular controls); mechanism decomposition kills the padding narrative and finds the arrival-uniformization artifact (-12.9%); cross-version churn surface shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held. Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer sidecars and metrics are tracked. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.328174978,
|
||||
"end_mono_ns": 199601637648726,
|
||||
"end_wall_ns": 1783868028764076133,
|
||||
"start_mono_ns": 199540309473748,
|
||||
"start_wall_ns": 1783867967435901551
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 39.29021051171873,
|
||||
"min": 4.445230456776771,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1787.3226249939762,
|
||||
"min": 37.47074800776318,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.25, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.25,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 143,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.471211524,
|
||||
"end_mono_ns": 199668842271892,
|
||||
"end_wall_ns": 1783868095968699176,
|
||||
"start_mono_ns": 199607371060368,
|
||||
"start_wall_ns": 1783868034497487431
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 143,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "932babe37b75c53f4a0eee029df66fd3e8acd3972d925f4a68714faa827b9366",
|
||||
"arrival_s": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"count": 143,
|
||||
"long_gt4096": 51,
|
||||
"offered_req_s": 2.3833333333333333,
|
||||
"offered_req_s_per_gpu": 2.3833333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 135,
|
||||
"finite_n": 143,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"raw_length_order_sha256": "eac9a2d39e762892f716fde086ada79aa87161d04819c24bbeaabbf9e8839af0",
|
||||
"request_id_order_sha256": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda"
|
||||
},
|
||||
"slo_pass_count": 143,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 44.43295162989699,
|
||||
"min": 4.492281684945301,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 143,
|
||||
"finite_n": 143,
|
||||
"max": 1784.063341008732,
|
||||
"min": 57.29520702152513,
|
||||
"missing_n": 0,
|
||||
"n": 143
|
||||
}
|
||||
}
|
||||
21
runs/opprof-phase6/phase6/cells/tp1_mns16/cell-valid.json
Normal file
21
runs/opprof-phase6/phase6/cells/tp1_mns16/cell-valid.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"accounting_mode": "A-P6-1-checkpoint-sidecar",
|
||||
"cell": "tp1_mns16",
|
||||
"invariants": {
|
||||
"all_anchor_intervals_covered": true,
|
||||
"all_schema_1": true,
|
||||
"checkpoint_after_all_anchor_intervals": true,
|
||||
"checkpoint_sidecar": true,
|
||||
"checkpoint_within_flush_of_stream": true,
|
||||
"complete_final_newline": true,
|
||||
"encoded_balanced": true,
|
||||
"last_step_matches": true,
|
||||
"no_in_stream_footer": true,
|
||||
"steps_contiguous": true,
|
||||
"two_anchor_intervals": true,
|
||||
"written_matches_records": true,
|
||||
"zero_drops": true
|
||||
},
|
||||
"layer1_records": 9212,
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2638886-1783867898065648658.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp1_mns16/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp1_mns16/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8501 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/warmup
|
||||
CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.24609375 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/anchor-0.24609375
|
||||
CLIENT taskset -c 20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns16 --anchor 0.25 --tp 1 --mns 16 --base-url http://127.0.0.1:8501 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/anchor-0.25
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638886-1783867898065648658.jsonl","encoded_records":9212,"written_records":9212,"dropped_records":0,"last_step_index":9211,"checkpoint_wall_ns":1783868096969027153,"flush_interval_seconds":1.0,"final":false}
|
||||
442
runs/opprof-phase6/phase6/cells/tp1_mns16/server.log
Normal file
442
runs/opprof-phase6/phase6/cells/tp1_mns16/server.log
Normal file
@@ -0,0 +1,442 @@
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8501, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 16, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2638187) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 32, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:05 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:37021 backend=nccl
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:05 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:01<00:15, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:02<00:14, 1.07s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:03<00:13, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:04<00:13, 1.10s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:05<00:11, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:06<00:10, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:07<00:09, 1.10s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:08<00:08, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:09<00:07, 1.08s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:10<00:06, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:11<00:05, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:12<00:04, 1.04s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:13<00:03, 1.06s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:15<00:02, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:16<00:01, 1.09s/it]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.17it/s]
|
||||
(EngineCore pid=2638886)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.03s/it]
|
||||
(EngineCore pid=2638886)
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.52 seconds
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.268242 seconds
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/f4a50989f8/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.31 s
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.252 s
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:31 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/fe5a82dbe929018b7aea7dee05b5cd31a21fe2682aca78ee4cbf2b37c8a086d6/rank_0_0/model
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:31 [monitor.py:53] torch.compile took 5.96 s in total
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:32 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.20 s
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:32 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.43 GiB
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 321,456 tokens
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.85x
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2638886) 2026-07-12 14:51:35,155 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2638886) 2026-07-12 14:51:35,199 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2638886)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 14%|█▍ | 1/7 [00:00<00:00, 9.98it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 43%|████▎ | 3/7 [00:00<00:00, 10.33it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 71%|███████▏ | 5/7 [00:00<00:00, 9.18it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▌ | 6/7 [00:00<00:00, 8.44it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00, 7.90it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:00<00:00, 8.49it/s]
|
||||
(EngineCore pid=2638886)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 40%|████ | 2/5 [00:00<00:00, 10.74it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 80%|████████ | 4/5 [00:00<00:00, 11.19it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 11.25it/s]
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:37 [gpu_model_runner.py:6660] Graph capturing finished in 2 secs, took 0.10 GiB
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:37 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.02 GiB (15.7%).
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:37 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:37 [core.py:337] init engine (profile, create kv cache, warmup model) took 11.83 s (compilation: 5.96 s)
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:38 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2638886-1783867898065648658.jsonl
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:38 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2638886) INFO 07-12 14:51:38 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2638187) WARNING 07-12 14:51:38 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8501
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2638187) INFO 07-12 14:51:38 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2638187) INFO: Started server process [2638187]
|
||||
(APIServer pid=2638187) INFO: Waiting for application startup.
|
||||
(APIServer pid=2638187) INFO: Application startup complete.
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52764 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638886) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638886) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:52808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:52:39 [loggers.py:273] Engine 000: Avg prompt throughput: 5299.4 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:52:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8.5 tokens/s, Avg generation throughput: 67.6 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 35.1%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:44254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:52:59 [loggers.py:273] Engine 000: Avg prompt throughput: 2139.7 tokens/s, Avg generation throughput: 267.4 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 43.4%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:38806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:45828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5435.1 tokens/s, Avg generation throughput: 320.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 33.3%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48094 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:19 [loggers.py:273] Engine 000: Avg prompt throughput: 4881.2 tokens/s, Avg generation throughput: 164.9 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.6%, Prefix cache hit rate: 29.1%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:48204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39864 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:29 [loggers.py:273] Engine 000: Avg prompt throughput: 6255.1 tokens/s, Avg generation throughput: 265.4 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.7%, Prefix cache hit rate: 24.2%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:39992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:39 [loggers.py:273] Engine 000: Avg prompt throughput: 9198.7 tokens/s, Avg generation throughput: 388.3 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.2%, Prefix cache hit rate: 20.8%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:40078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42934 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:42996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:43012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:43016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:43018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:43030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:49 [loggers.py:273] Engine 000: Avg prompt throughput: 6382.3 tokens/s, Avg generation throughput: 331.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.1%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53630 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:53:59 [loggers.py:273] Engine 000: Avg prompt throughput: 3970.7 tokens/s, Avg generation throughput: 126.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.3%, Prefix cache hit rate: 17.8%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:53988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54088 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:09 [loggers.py:273] Engine 000: Avg prompt throughput: 6063.3 tokens/s, Avg generation throughput: 290.7 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.9%, Prefix cache hit rate: 17.1%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54104 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:54120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:19 [loggers.py:273] Engine 000: Avg prompt throughput: 4203.4 tokens/s, Avg generation throughput: 273.5 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 16.7%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56492 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:56502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34920 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34938 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:34994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:29 [loggers.py:273] Engine 000: Avg prompt throughput: 4720.8 tokens/s, Avg generation throughput: 256.7 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 16.3%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:35028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57030 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:39 [loggers.py:273] Engine 000: Avg prompt throughput: 8295.5 tokens/s, Avg generation throughput: 280.7 tokens/s, Running: 16 reqs, Waiting: 1 reqs, GPU KV cache usage: 15.5%, Prefix cache hit rate: 15.5%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:57276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36812 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36824 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36838 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36922 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO 07-12 14:54:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8540.1 tokens/s, Avg generation throughput: 333.9 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 17.1%, Prefix cache hit rate: 14.6%
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638187) INFO: 127.0.0.1:36962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp1_mns16/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp1_mns16/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns16", "anchor": 0.24609375, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp1_mns16/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp1_mns16/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.031388113,
|
||||
"end_mono_ns": 199530259142500,
|
||||
"end_wall_ns": 1783867957385569525,
|
||||
"start_mono_ns": 199522227754387,
|
||||
"start_wall_ns": 1783867949354181463
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 16,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 28.370322086701652,
|
||||
"min": 4.984751267873821,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 696.6323160158936,
|
||||
"min": 74.85444800113328,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.2421875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 137,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.294849452,
|
||||
"end_mono_ns": 199601604247251,
|
||||
"end_wall_ns": 1783868028730674502,
|
||||
"start_mono_ns": 199540309397799,
|
||||
"start_wall_ns": 1783867967435824912
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 137,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "e8e91fd47d9152811ed7bd79e20c0f45ba6677560a419542d9c17abd82bebb4b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"count": 137,
|
||||
"long_gt4096": 47,
|
||||
"offered_req_s": 2.283333333333333,
|
||||
"offered_req_s_per_gpu": 2.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 137,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"raw_length_order_sha256": "93176915562ff9a118f3550157ddd1025d73a6b70cf4d03be9765de9a1b5d744",
|
||||
"request_id_order_sha256": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017"
|
||||
},
|
||||
"slo_pass_count": 137,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 35.17025839386134,
|
||||
"min": 4.435019960625127,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 1485.6346309825312,
|
||||
"min": 34.36101900297217,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.288241408,
|
||||
"end_mono_ns": 199668653603418,
|
||||
"end_wall_ns": 1783868095780030817,
|
||||
"start_mono_ns": 199607365362010,
|
||||
"start_wall_ns": 1783868034491789246
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 45.32469625197036,
|
||||
"min": 4.495332385807511,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1799.771893012803,
|
||||
"min": 49.012041999958456,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
21
runs/opprof-phase6/phase6/cells/tp1_mns32/cell-valid.json
Normal file
21
runs/opprof-phase6/phase6/cells/tp1_mns32/cell-valid.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"accounting_mode": "A-P6-1-checkpoint-sidecar",
|
||||
"cell": "tp1_mns32",
|
||||
"invariants": {
|
||||
"all_anchor_intervals_covered": true,
|
||||
"all_schema_1": true,
|
||||
"checkpoint_after_all_anchor_intervals": true,
|
||||
"checkpoint_sidecar": true,
|
||||
"checkpoint_within_flush_of_stream": true,
|
||||
"complete_final_newline": true,
|
||||
"encoded_balanced": true,
|
||||
"last_step_matches": true,
|
||||
"no_in_stream_footer": true,
|
||||
"steps_contiguous": true,
|
||||
"two_anchor_intervals": true,
|
||||
"written_matches_records": true,
|
||||
"zero_drops": true
|
||||
},
|
||||
"layer1_records": 9633,
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2638900-1783867898685556703.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp1_mns32/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp1_mns32/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8502 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 32 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.2421875 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/warmup
|
||||
CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.2421875 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/anchor-0.2421875
|
||||
CLIENT taskset -c 40-59 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns32 --anchor 0.24609375 --tp 1 --mns 32 --base-url http://127.0.0.1:8502 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/anchor-0.24609375
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638900-1783867898685556703.jsonl","encoded_records":9633,"written_records":9633,"dropped_records":0,"last_step_index":9632,"checkpoint_wall_ns":1783868096780742186,"flush_interval_seconds":1.0,"final":false}
|
||||
436
runs/opprof-phase6/phase6/cells/tp1_mns32/server.log
Normal file
436
runs/opprof-phase6/phase6/cells/tp1_mns32/server.log
Normal file
@@ -0,0 +1,436 @@
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8502, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 32, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2638188) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:05 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:50247 backend=nccl
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:05 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:01<00:16, 1.07s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:02<00:14, 1.07s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:03<00:13, 1.06s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:04<00:13, 1.10s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:05<00:11, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:06<00:10, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:07<00:09, 1.10s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:08<00:08, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:09<00:07, 1.08s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:10<00:06, 1.06s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:11<00:05, 1.06s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:12<00:04, 1.04s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:13<00:03, 1.06s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:15<00:02, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:16<00:01, 1.09s/it]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.18it/s]
|
||||
(EngineCore pid=2638900)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.03s/it]
|
||||
(EngineCore pid=2638900)
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.49 seconds
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.231743 seconds
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/30ef41b5f5/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.33 s
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.266 s
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:32 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/738c624149a63d13eaf115eec4d2189ece948ac500e524d3e06e801d9915352d/rank_0_0/model
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:32 [monitor.py:53] torch.compile took 6.02 s in total
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:32 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.20 s
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:33 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=11 (largest=64), FULL=7 (largest=32)
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.11 GiB total
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.4 GiB
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9188 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9212. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 321,136 tokens
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.84x
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2638900) 2026-07-12 14:51:35,118 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2638900) 2026-07-12 14:51:35,162 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2638900)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 9%|▉ | 1/11 [00:00<00:01, 9.36it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 18%|█▊ | 2/11 [00:00<00:00, 9.41it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 3/11 [00:00<00:00, 9.30it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 36%|███▋ | 4/11 [00:00<00:00, 9.23it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 5/11 [00:00<00:00, 9.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 55%|█████▍ | 6/11 [00:00<00:00, 9.24it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 64%|██████▎ | 7/11 [00:00<00:00, 8.56it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 73%|███████▎ | 8/11 [00:00<00:00, 8.19it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 9/11 [00:01<00:00, 7.88it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 91%|█████████ | 10/11 [00:01<00:00, 7.84it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:01<00:00, 7.60it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:01<00:00, 8.31it/s]
|
||||
(EngineCore pid=2638900)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 2/7 [00:00<00:00, 10.42it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 57%|█████▋ | 4/7 [00:00<00:00, 10.82it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 86%|████████▌ | 6/7 [00:00<00:00, 10.80it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 7/7 [00:00<00:00, 10.91it/s]
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:37 [gpu_model_runner.py:6660] Graph capturing finished in 3 secs, took 0.13 GiB
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:37 [gpu_worker.py:667] CUDA graph pool memory: 0.13 GiB (actual), 0.11 GiB (estimated), difference: 0.02 GiB (12.1%).
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:37 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:38 [core.py:337] init engine (profile, create kv cache, warmup model) took 12.52 s (compilation: 6.02 s)
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:38 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2638900-1783867898685556703.jsonl
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:38 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2638900) INFO 07-12 14:51:38 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:38 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2638188) WARNING 07-12 14:51:38 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8502
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2638188) INFO 07-12 14:51:39 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2638188) INFO: Started server process [2638188]
|
||||
(APIServer pid=2638188) INFO: Waiting for application startup.
|
||||
(APIServer pid=2638188) INFO: Application startup complete.
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43588 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638900) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638900) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:43626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59190 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59200 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59238 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59254 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:52:39 [loggers.py:273] Engine 000: Avg prompt throughput: 5299.4 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:52:49 [loggers.py:273] Engine 000: Avg prompt throughput: 8.5 tokens/s, Avg generation throughput: 95.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.1%, Prefix cache hit rate: 35.1%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:52:59 [loggers.py:273] Engine 000: Avg prompt throughput: 3058.0 tokens/s, Avg generation throughput: 240.8 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.5%, Prefix cache hit rate: 40.3%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:42340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5130.2 tokens/s, Avg generation throughput: 333.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 33.3%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38162 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:19 [loggers.py:273] Engine 000: Avg prompt throughput: 3991.7 tokens/s, Avg generation throughput: 168.0 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.7%, Prefix cache hit rate: 29.4%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:38250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55960 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:55992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:29 [loggers.py:273] Engine 000: Avg prompt throughput: 6239.1 tokens/s, Avg generation throughput: 226.6 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.3%, Prefix cache hit rate: 24.2%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:56140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33508 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:39 [loggers.py:273] Engine 000: Avg prompt throughput: 7873.3 tokens/s, Avg generation throughput: 414.1 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.4%, Prefix cache hit rate: 21.7%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:59666 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:49 [loggers.py:273] Engine 000: Avg prompt throughput: 5848.7 tokens/s, Avg generation throughput: 274.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.8%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:53:59 [loggers.py:273] Engine 000: Avg prompt throughput: 4474.5 tokens/s, Avg generation throughput: 142.1 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.5%, Prefix cache hit rate: 18.4%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36498 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:36594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:54:09 [loggers.py:273] Engine 000: Avg prompt throughput: 5897.7 tokens/s, Avg generation throughput: 295.1 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.9%, Prefix cache hit rate: 17.5%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:45994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:54:19 [loggers.py:273] Engine 000: Avg prompt throughput: 3893.6 tokens/s, Avg generation throughput: 269.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 17.2%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:46150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48862 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:54:29 [loggers.py:273] Engine 000: Avg prompt throughput: 4692.9 tokens/s, Avg generation throughput: 265.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.7%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:48924 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39894 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39916 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:39992 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40054 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO 07-12 14:54:39 [loggers.py:273] Engine 000: Avg prompt throughput: 8338.1 tokens/s, Avg generation throughput: 294.5 tokens/s, Running: 17 reqs, Waiting: 0 reqs, GPU KV cache usage: 15.7%, Prefix cache hit rate: 16.0%
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:40114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33368 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638188) INFO: 127.0.0.1:33384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp1_mns32/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp1_mns32/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns32", "anchor": 0.2421875, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp1_mns32/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp1_mns32/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.025285552,
|
||||
"end_mono_ns": 199530264041528,
|
||||
"end_wall_ns": 1783867957390468387,
|
||||
"start_mono_ns": 199522238755976,
|
||||
"start_wall_ns": 1783867949365182961
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 32,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 28.123532992047,
|
||||
"min": 4.953902173286861,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 682.8775229805615,
|
||||
"min": 74.39436702406965,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.2421875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 137,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.27871102,
|
||||
"end_mono_ns": 199601593594258,
|
||||
"end_wall_ns": 1783868028720022637,
|
||||
"start_mono_ns": 199540314883238,
|
||||
"start_wall_ns": 1783867967441310423
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 137,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "e8e91fd47d9152811ed7bd79e20c0f45ba6677560a419542d9c17abd82bebb4b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"count": 137,
|
||||
"long_gt4096": 47,
|
||||
"offered_req_s": 2.283333333333333,
|
||||
"offered_req_s_per_gpu": 2.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 137,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"raw_length_order_sha256": "93176915562ff9a118f3550157ddd1025d73a6b70cf4d03be9765de9a1b5d744",
|
||||
"request_id_order_sha256": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017"
|
||||
},
|
||||
"slo_pass_count": 137,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 34.5918034883051,
|
||||
"min": 4.451991834606257,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 137,
|
||||
"finite_n": 137,
|
||||
"max": 1472.6779730117414,
|
||||
"min": 33.561496005859226,
|
||||
"missing_n": 0,
|
||||
"n": 137
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.24609375, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.24609375,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 141,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.283849775,
|
||||
"end_mono_ns": 199668751241940,
|
||||
"end_wall_ns": 1783868095877669237,
|
||||
"start_mono_ns": 199607467392165,
|
||||
"start_wall_ns": 1783868034593819130
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 141,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "7c8f4ce3fc2db40d6329e8ead59378f564608ad6df09bc95854baef2dd0703d4",
|
||||
"arrival_s": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"count": 141,
|
||||
"long_gt4096": 50,
|
||||
"offered_req_s": 2.35,
|
||||
"offered_req_s_per_gpu": 2.35,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 133,
|
||||
"finite_n": 141,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"raw_length_order_sha256": "a30d27aea9630ed3623c73237867fbd3a6b7eec9bedec0f1c390610fd16ea5ba",
|
||||
"request_id_order_sha256": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3"
|
||||
},
|
||||
"slo_pass_count": 141,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 48.397922928960206,
|
||||
"min": 4.49719877962177,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 141,
|
||||
"finite_n": 141,
|
||||
"max": 1777.0718620158732,
|
||||
"min": 49.70270599005744,
|
||||
"missing_n": 0,
|
||||
"n": 141
|
||||
}
|
||||
}
|
||||
21
runs/opprof-phase6/phase6/cells/tp1_mns64/cell-valid.json
Normal file
21
runs/opprof-phase6/phase6/cells/tp1_mns64/cell-valid.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"accounting_mode": "A-P6-1-checkpoint-sidecar",
|
||||
"cell": "tp1_mns64",
|
||||
"invariants": {
|
||||
"all_anchor_intervals_covered": true,
|
||||
"all_schema_1": true,
|
||||
"checkpoint_after_all_anchor_intervals": true,
|
||||
"checkpoint_sidecar": true,
|
||||
"checkpoint_within_flush_of_stream": true,
|
||||
"complete_final_newline": true,
|
||||
"encoded_balanced": true,
|
||||
"last_step_matches": true,
|
||||
"no_in_stream_footer": true,
|
||||
"steps_contiguous": true,
|
||||
"two_anchor_intervals": true,
|
||||
"written_matches_records": true,
|
||||
"zero_drops": true
|
||||
},
|
||||
"layer1_records": 9654,
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2638898-1783867899798067669.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp1_mns64/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp1_mns64/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8503 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 64 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.2421875 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/warmup
|
||||
CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.2421875 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/anchor-0.2421875
|
||||
CLIENT taskset -c 60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns64 --anchor 0.24609375 --tp 1 --mns 64 --base-url http://127.0.0.1:8503 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/anchor-0.24609375
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638898-1783867899798067669.jsonl","encoded_records":9654,"written_records":9654,"dropped_records":0,"last_step_index":9653,"checkpoint_wall_ns":1783868097891402992,"flush_interval_seconds":1.0,"final":false}
|
||||
437
runs/opprof-phase6/phase6/cells/tp1_mns64/server.log
Normal file
437
runs/opprof-phase6/phase6/cells/tp1_mns64/server.log
Normal file
@@ -0,0 +1,437 @@
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8503, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 64, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2638189) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:06 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:39471 backend=nccl
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:06 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:01<00:15, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:02<00:14, 1.07s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:03<00:13, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:04<00:13, 1.10s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:05<00:11, 1.08s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:06<00:10, 1.09s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:07<00:09, 1.11s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:08<00:08, 1.09s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:09<00:07, 1.08s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:10<00:06, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:11<00:05, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:12<00:04, 1.04s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:13<00:03, 1.06s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:15<00:02, 1.09s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:16<00:01, 1.09s/it]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.17it/s]
|
||||
(EngineCore pid=2638898)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.03s/it]
|
||||
(EngineCore pid=2638898)
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.52 seconds
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.262335 seconds
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:29 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/65b50dadd2/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:29 [backends.py:1148] Dynamo bytecode transform time: 3.30 s
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:31 [backends.py:292] Directly load the compiled graph(s) for compile range (1, 8192) from the cache, took 2.131 s
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:31 [decorators.py:311] Directly load AOT compilation from path /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/ab53f49ec98f407fe24fecb037cb59739264f283939c70f41986d8369686d472/rank_0_0/model
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:31 [monitor.py:53] torch.compile took 5.83 s in total
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:31 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:32 [monitor.py:81] Initial profiling/warmup run took 0.21 s
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:32 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=19 (largest=128), FULL=11 (largest=64)
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.19 GiB total
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_worker.py:508] Available KV cache memory: 29.33 GiB
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9180 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9220. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [kv_cache_utils.py:2146] GPU KV cache size: 320,304 tokens
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.82x
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2638898) 2026-07-12 14:51:35,044 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2638898) 2026-07-12 14:51:35,116 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2638898)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:01, 9.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:00<00:01, 10.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:00<00:01, 10.01it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:00<00:01, 9.67it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:00<00:01, 9.52it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:01<00:00, 9.72it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:01<00:00, 9.69it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:01<00:00, 9.19it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:01<00:00, 9.33it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:01<00:00, 8.79it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:01<00:00, 8.44it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:01<00:00, 8.20it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:01<00:00, 7.99it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:02<00:00, 8.05it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:02<00:00, 7.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:02<00:00, 8.89it/s]
|
||||
(EngineCore pid=2638898)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:00, 10.99it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:00<00:00, 11.14it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:00<00:00, 11.17it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:00<00:00, 11.23it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:00<00:00, 11.31it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:00<00:00, 11.28it/s]
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:38 [gpu_model_runner.py:6660] Graph capturing finished in 4 secs, took 0.24 GiB
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:38 [gpu_worker.py:667] CUDA graph pool memory: 0.24 GiB (actual), 0.19 GiB (estimated), difference: 0.05 GiB (19.8%).
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:38 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:39 [core.py:337] init engine (profile, create kv cache, warmup model) took 13.60 s (compilation: 5.83 s)
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:39 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2638898-1783867899798067669.jsonl
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:39 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2638898) INFO 07-12 14:51:39 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:39 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2638189) WARNING 07-12 14:51:40 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8503
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2638189) INFO 07-12 14:51:40 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2638189) INFO: Started server process [2638189]
|
||||
(APIServer pid=2638189) INFO: Waiting for application startup.
|
||||
(APIServer pid=2638189) INFO: Application startup complete.
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46204 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638898) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638898) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:52:30 [loggers.py:273] Engine 000: Avg prompt throughput: 1668.3 tokens/s, Avg generation throughput: 1.5 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.6%, Prefix cache hit rate: 4.0%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:54256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:52:40 [loggers.py:273] Engine 000: Avg prompt throughput: 3630.8 tokens/s, Avg generation throughput: 203.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:39452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:52:50 [loggers.py:273] Engine 000: Avg prompt throughput: 11.3 tokens/s, Avg generation throughput: 117.9 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 42.7%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46430 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46612 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:46636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:00 [loggers.py:273] Engine 000: Avg prompt throughput: 3820.4 tokens/s, Avg generation throughput: 268.8 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 40.3%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33684 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:33836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:10 [loggers.py:273] Engine 000: Avg prompt throughput: 4364.6 tokens/s, Avg generation throughput: 291.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.3%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51684 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51766 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51772 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:51834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:20 [loggers.py:273] Engine 000: Avg prompt throughput: 4686.8 tokens/s, Avg generation throughput: 194.6 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 28.9%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55514 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55570 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55574 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55588 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:55658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:30 [loggers.py:273] Engine 000: Avg prompt throughput: 7377.9 tokens/s, Avg generation throughput: 242.3 tokens/s, Running: 15 reqs, Waiting: 0 reqs, GPU KV cache usage: 15.3%, Prefix cache hit rate: 24.2%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37740 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:40 [loggers.py:273] Engine 000: Avg prompt throughput: 6039.0 tokens/s, Avg generation throughput: 407.8 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 21.3%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52784 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52794 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52816 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:52840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:53:50 [loggers.py:273] Engine 000: Avg prompt throughput: 5848.5 tokens/s, Avg generation throughput: 230.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 19.8%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:53446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:00 [loggers.py:273] Engine 000: Avg prompt throughput: 4481.1 tokens/s, Avg generation throughput: 175.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.1%, Prefix cache hit rate: 18.4%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42282 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:42428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:10 [loggers.py:273] Engine 000: Avg prompt throughput: 6653.1 tokens/s, Avg generation throughput: 325.3 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 17.6%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43200 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43234 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:43288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:20 [loggers.py:273] Engine 000: Avg prompt throughput: 3169.4 tokens/s, Avg generation throughput: 228.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 17.3%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:37330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:30 [loggers.py:273] Engine 000: Avg prompt throughput: 4896.1 tokens/s, Avg generation throughput: 259.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 16.9%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:59996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:60124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO 07-12 14:54:40 [loggers.py:273] Engine 000: Avg prompt throughput: 8725.1 tokens/s, Avg generation throughput: 309.3 tokens/s, Running: 18 reqs, Waiting: 0 reqs, GPU KV cache usage: 19.8%, Prefix cache hit rate: 15.8%
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638189) INFO: 127.0.0.1:48462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp1_mns64/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp1_mns64/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns64", "anchor": 0.2421875, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp1_mns64/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp1_mns64/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2421875,
|
||||
"cell": "tp1_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 8.029548257,
|
||||
"end_mono_ns": 199530257119882,
|
||||
"end_wall_ns": 1783867957383546922,
|
||||
"start_mono_ns": 199522227571625,
|
||||
"start_wall_ns": 1783867949353998820
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 64,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1bfb7b673b63b8aaea00c075792180307e1a32e354711a8d3c4978f9a013bc02",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 6.901699999999983,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 8,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "e858719eb07cdeb674aa17d9299d05dec852db11c263bc9391c53cd0f177db1c",
|
||||
"request_id_order_sha256": "0fa4b7f4dc274280eb173a0ee0974643ab283b887418ce8f10e958e7e8d90161"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 28.450962196848554,
|
||||
"min": 4.971639937024534,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 697.7603349951096,
|
||||
"min": 52.274041023338214,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.21875, "kind": "anchor", "pass_rate": 0.9917355371900827, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.21875,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 121,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.294286331,
|
||||
"end_mono_ns": 199668659416799,
|
||||
"end_wall_ns": 1783868095785843711,
|
||||
"start_mono_ns": 199607365130468,
|
||||
"start_wall_ns": 1783868034491558052
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 121,
|
||||
"pass_rate": 0.9917355371900827,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "f585e4793a8a6a621dbfe622514743ac316e34e3e93aaa1c6ba5f1eb58e8ff2d",
|
||||
"arrival_s": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"count": 121,
|
||||
"long_gt4096": 42,
|
||||
"offered_req_s": 2.0166666666666666,
|
||||
"offered_req_s_per_gpu": 2.0166666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 114,
|
||||
"finite_n": 121,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"raw_length_order_sha256": "061592e591871368954f4f80ee75cde2cfc399bd719f3f5deaad275e9af72e8b",
|
||||
"request_id_order_sha256": "9b8c177ea00e5e42bd3294328201cb57addfd823913a5721714ce69646d715f4"
|
||||
},
|
||||
"slo_pass_count": 120,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 30.344350779553743,
|
||||
"min": 4.4550072992088525,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 121,
|
||||
"finite_n": 121,
|
||||
"max": 2038.4164949937258,
|
||||
"min": 37.96417900593951,
|
||||
"missing_n": 0,
|
||||
"n": 121
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "anchor", "pass_rate": 0.20634920634920634, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2265625,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 48,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 28.662121521,
|
||||
"end_mono_ns": 199568971776020,
|
||||
"end_wall_ns": 1783867996098203014,
|
||||
"start_mono_ns": 199540309654499,
|
||||
"start_wall_ns": 1783867967436081499
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 126,
|
||||
"pass_rate": 0.20634920634920634,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "1b67241583bd6f75b1ad3f6d8c86bf0c815306a28f18ea49b1511fed6c091abe",
|
||||
"arrival_s": {
|
||||
"distinct_n": 126,
|
||||
"finite_n": 126,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"count": 126,
|
||||
"long_gt4096": 44,
|
||||
"offered_req_s": 2.1,
|
||||
"offered_req_s_per_gpu": 2.1,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 119,
|
||||
"finite_n": 126,
|
||||
"max": 8149.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 126
|
||||
},
|
||||
"raw_length_order_sha256": "29e3240c20df0cc68172d39aeae84d79908acf564c1b1e2edd447a984e7bdb0e",
|
||||
"request_id_order_sha256": "a05848006335fc9f595432bab793a13d427244886e4f04e51ba74d495453ef90"
|
||||
},
|
||||
"slo_pass_count": 26,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 48,
|
||||
"finite_n": 48,
|
||||
"max": 48.57467018882744,
|
||||
"min": 4.850139889790948,
|
||||
"missing_n": 78,
|
||||
"n": 126
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 48,
|
||||
"finite_n": 48,
|
||||
"max": 7013.455057982355,
|
||||
"min": 39.39470701152459,
|
||||
"missing_n": 78,
|
||||
"n": 126
|
||||
}
|
||||
}
|
||||
21
runs/opprof-phase6/phase6/cells/tp1_mns8/cell-valid.json
Normal file
21
runs/opprof-phase6/phase6/cells/tp1_mns8/cell-valid.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"accounting_mode": "A-P6-1-checkpoint-sidecar",
|
||||
"cell": "tp1_mns8",
|
||||
"invariants": {
|
||||
"all_anchor_intervals_covered": true,
|
||||
"all_schema_1": true,
|
||||
"checkpoint_after_all_anchor_intervals": true,
|
||||
"checkpoint_sidecar": true,
|
||||
"checkpoint_within_flush_of_stream": true,
|
||||
"complete_final_newline": true,
|
||||
"encoded_balanced": true,
|
||||
"last_step_matches": true,
|
||||
"no_in_stream_footer": true,
|
||||
"steps_contiguous": true,
|
||||
"two_anchor_intervals": true,
|
||||
"written_matches_records": true,
|
||||
"zero_drops": true
|
||||
},
|
||||
"layer1_records": 7683,
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2638884-1783867942846550713.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp1_mns8/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp1_mns8/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8500 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 8 --tensor-parallel-size 1 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/warmup
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.2265625 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/anchor-0.2265625
|
||||
CLIENT taskset -c 0-19 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp1_mns8 --anchor 0.21875 --tp 1 --mns 8 --base-url http://127.0.0.1:8500 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/anchor-0.21875
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2638884-1783867942846550713.jsonl","encoded_records":7683,"written_records":7683,"dropped_records":0,"last_step_index":7682,"checkpoint_wall_ns":1783868096786407319,"flush_interval_seconds":1.0,"final":false}
|
||||
327
runs/opprof-phase6/phase6/cells/tp1_mns8/server.log
Normal file
327
runs/opprof-phase6/phase6/cells/tp1_mns8/server.log
Normal file
@@ -0,0 +1,327 @@
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:339]
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:41 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8500, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'max_num_batched_tokens': 8192, 'max_num_seqs': 8, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2638186) INFO 07-12 14:50:52 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:03 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 16, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:06 [parallel_state.py:1588] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.27.132.244:48769 backend=nccl
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:06 [parallel_state.py:1923] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:07 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:08 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1283.36 GiB.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:08 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:01<00:15, 1.06s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:02<00:14, 1.07s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:03<00:13, 1.06s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:04<00:13, 1.10s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:05<00:11, 1.07s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:06<00:10, 1.09s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:07<00:09, 1.11s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:08<00:08, 1.10s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:09<00:07, 1.08s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:10<00:06, 1.07s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:11<00:05, 1.06s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:12<00:04, 1.04s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:13<00:03, 1.06s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:15<00:02, 1.09s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:16<00:01, 1.09s/it]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.18it/s]
|
||||
(EngineCore pid=2638884)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:16<00:00, 1.03s/it]
|
||||
(EngineCore pid=2638884)
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:25 [default_loader.py:430] Loading weights took 16.49 seconds
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:25 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:25 [gpu_model_runner.py:5259] Model loading took 56.88 GiB memory and 17.240093 seconds
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:35 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/b95eb69a3d/rank_0_0/backbone for vLLM's torch.compile
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:35 [backends.py:1148] Dynamo bytecode transform time: 9.16 s
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:50 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
|
||||
(EngineCore pid=2638884) INFO 07-12 14:51:58 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 23.00 s
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:03 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/802e9eed10a34f87d59e9a1acfc3b883a63a5eaeb40312e4832ac27dfbf354e5/rank_0_0/model
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:03 [monitor.py:53] torch.compile took 37.35 s in total
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:03 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=768,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:05 [monitor.py:81] Initial profiling/warmup run took 2.19 s
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:12 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:17 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [gpu_worker.py:508] Available KV cache memory: 29.44 GiB
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [kv_cache_utils.py:2146] GPU KV cache size: 321,600 tokens
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 7.85x
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:18 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(EngineCore pid=2638884) 2026-07-12 14:52:18,390 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(EngineCore pid=2638884) 2026-07-12 14:52:18,429 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(EngineCore pid=2638884)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 40%|████ | 2/5 [00:00<00:00, 10.98it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 4/5 [00:01<00:00, 2.57it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 1.77it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 2.12it/s]
|
||||
(EngineCore pid=2638884)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/4 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 50%|█████ | 2/4 [00:00<00:00, 11.61it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 11.88it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 11.84it/s]
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:21 [gpu_model_runner.py:6660] Graph capturing finished in 3 secs, took 0.08 GiB
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:21 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (12.2%).
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:21 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:22 [core.py:337] init engine (profile, create kv cache, warmup model) took 56.63 s (compilation: 37.35 s)
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:22 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2638884-1783867942846550713.jsonl
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:22 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2638884) INFO 07-12 14:52:22 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:22 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2638186) WARNING 07-12 14:52:23 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8500
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:23 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2638186) INFO: Started server process [2638186]
|
||||
(APIServer pid=2638186) INFO: Waiting for application startup.
|
||||
(APIServer pid=2638186) INFO: Application startup complete.
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:50438 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:50454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:50460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638884) WARNING 07-12 14:52:29 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:50472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(EngineCore pid=2638884) WARNING 07-12 14:52:31 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:33 [loggers.py:273] Engine 000: Avg prompt throughput: 1338.9 tokens/s, Avg generation throughput: 0.9 tokens/s, Running: 8 reqs, Waiting: 1 reqs, GPU KV cache usage: 7.1%, Prefix cache hit rate: 4.2%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:43 [loggers.py:273] Engine 000: Avg prompt throughput: 2863.4 tokens/s, Avg generation throughput: 203.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 5.0%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34542 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:34572 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:52:53 [loggers.py:273] Engine 000: Avg prompt throughput: 14.3 tokens/s, Avg generation throughput: 153.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 47.0%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51346 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51422 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:51438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:53:03 [loggers.py:273] Engine 000: Avg prompt throughput: 3326.2 tokens/s, Avg generation throughput: 188.1 tokens/s, Running: 7 reqs, Waiting: 8 reqs, GPU KV cache usage: 7.0%, Prefix cache hit rate: 39.8%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42476 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:42522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:53:13 [loggers.py:273] Engine 000: Avg prompt throughput: 3070.5 tokens/s, Avg generation throughput: 160.1 tokens/s, Running: 7 reqs, Waiting: 3 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 34.1%
|
||||
(APIServer pid=2638186) INFO 07-12 14:53:23 [loggers.py:273] Engine 000: Avg prompt throughput: 674.1 tokens/s, Avg generation throughput: 112.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.4%
|
||||
(APIServer pid=2638186) INFO 07-12 14:53:33 [loggers.py:273] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 33.4%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35694 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35714 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35656 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35670 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35690 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:03 [loggers.py:273] Engine 000: Avg prompt throughput: 20.3 tokens/s, Avg generation throughput: 219.0 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 47.8%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35722 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35732 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35736 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35780 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35806 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:35836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59738 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59744 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:13 [loggers.py:273] Engine 000: Avg prompt throughput: 21.0 tokens/s, Avg generation throughput: 311.2 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 58.8%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59770 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:59822 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36438 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:23 [loggers.py:273] Engine 000: Avg prompt throughput: 1393.0 tokens/s, Avg generation throughput: 110.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 56.5%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36480 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36546 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:36558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58042 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5941.7 tokens/s, Avg generation throughput: 269.5 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 48.4%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58122 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58138 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58176 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58206 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58222 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:58228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47550 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47554 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47564 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO 07-12 14:54:43 [loggers.py:273] Engine 000: Avg prompt throughput: 8113.3 tokens/s, Avg generation throughput: 349.9 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 41.3%
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47602 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47612 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47664 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:47708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2638186) INFO: 127.0.0.1:39082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp1_mns8/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp1_mns8/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp1_mns8", "anchor": 0.2265625, "kind": "warmup", "pass_rate": 0.3125, "feasible": false}
|
||||
74
runs/opprof-phase6/phase6/cells/tp1_mns8/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp1_mns8/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.2265625,
|
||||
"cell": "tp1_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 11.581101349,
|
||||
"end_mono_ns": 199533809459936,
|
||||
"end_wall_ns": 1783867960935887048,
|
||||
"start_mono_ns": 199522228358587,
|
||||
"start_wall_ns": 1783867949354785531
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 8,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 0.3125,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "899b928ee7acd2abc5bf86f6641591111d09f06ec3dbf35a687067b5dfe9ad57",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7.4525000000000095,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 6,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.26666666666666666,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 72.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "c99abf7970483c5386d5c34fad5a1b2664c29b8363b5b6e096338ed8e9773d27",
|
||||
"request_id_order_sha256": "9a7cbf2821c1a7b2a498858815a6f31b0d307f2c8a4900d8be107e6a15858964"
|
||||
},
|
||||
"slo_pass_count": 5,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 1,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 37.95462277145895,
|
||||
"min": 10.364419787317354,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 4013.1819479865953,
|
||||
"min": 2348.2964480062947,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.4921875, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.4921875,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 269,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.339108631,
|
||||
"end_mono_ns": 201223962200316,
|
||||
"end_wall_ns": 1783869651088627469,
|
||||
"start_mono_ns": 201162623091685,
|
||||
"start_wall_ns": 1783869589749518962
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 269,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "ffbc7b8dd324c8e745654110a2412b03ddade891beb2bfe022ffde6502b0184b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"count": 269,
|
||||
"long_gt4096": 95,
|
||||
"offered_req_s": 4.483333333333333,
|
||||
"offered_req_s_per_gpu": 2.2416666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 250,
|
||||
"finite_n": 269,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"raw_length_order_sha256": "bc951350376a487d34d2def78fd32a2286e23bf8a7f481ff578009e12e3f1fb8",
|
||||
"request_id_order_sha256": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c"
|
||||
},
|
||||
"slo_pass_count": 269,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 37.621230566938316,
|
||||
"min": 5.02581280306913,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 1802.1751450141892,
|
||||
"min": 38.09416797594167,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.49609375, "kind": "anchor", "pass_rate": 0.08791208791208792, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 85,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 29.202033933,
|
||||
"end_mono_ns": 201151639670622,
|
||||
"end_wall_ns": 1783869578766097690,
|
||||
"start_mono_ns": 201122437636689,
|
||||
"start_wall_ns": 1783869549564063872
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 273,
|
||||
"pass_rate": 0.08791208791208792,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "41168f4bd02f3ef4e83b7440a3dab7458f08df9c2bbd4452fad076b4a2e0eaed",
|
||||
"arrival_s": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"count": 273,
|
||||
"long_gt4096": 97,
|
||||
"offered_req_s": 4.55,
|
||||
"offered_req_s_per_gpu": 2.275,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 253,
|
||||
"finite_n": 273,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"raw_length_order_sha256": "9e38ed1766e3933dc05a7ced5d0a73fed47ce769800f0d2c10014af0ef823f28",
|
||||
"request_id_order_sha256": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80"
|
||||
},
|
||||
"slo_pass_count": 24,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 85,
|
||||
"finite_n": 85,
|
||||
"max": 56.565095488335864,
|
||||
"min": 5.2603521261925215,
|
||||
"missing_n": 188,
|
||||
"n": 273
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 85,
|
||||
"finite_n": 85,
|
||||
"max": 11585.765292984433,
|
||||
"min": 40.21007599658333,
|
||||
"missing_n": 188,
|
||||
"n": 273
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp2_mns16/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp2_mns16/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns16",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 6313,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/opprof/opprof-v1-dp0-pid2661159-1783869522696890561.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp2_mns16/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp2_mns16/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 40-59,60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8511 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 40-59,60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.49609375 --tp 2 --mns 16 --base-url http://127.0.0.1:8511 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/warmup
|
||||
CLIENT taskset -c 40-59,60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.49609375 --tp 2 --mns 16 --base-url http://127.0.0.1:8511 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/anchor-0.49609375
|
||||
CLIENT taskset -c 40-59,60-79 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns16 --anchor 0.4921875 --tp 2 --mns 16 --base-url http://127.0.0.1:8511 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/anchor-0.4921875
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661159-1783869522696890561.jsonl","encoded_records":6313,"written_records":6313,"dropped_records":0,"last_step_index":6312,"checkpoint_wall_ns":1783869798733332776,"flush_interval_seconds":1.0,"final":true}
|
||||
538
runs/opprof-phase6/phase6/cells/tp2_mns16/server.log
Normal file
538
runs/opprof-phase6/phase6/cells/tp2_mns16/server.log
Normal file
@@ -0,0 +1,538 @@
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8511, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'tensor_parallel_size': 2, 'max_num_batched_tokens': 8192, 'max_num_seqs': 16, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:40 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2660455) INFO 07-12 15:16:43 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(EngineCore pid=2661159) INFO 07-12 15:16:53 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': True, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 32, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2661159) WARNING 07-12 15:16:53 [multiproc_executor.py:1063] Reducing Torch parallelism from 40 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
(EngineCore pid=2661159) INFO 07-12 15:16:53 [multiproc_executor.py:140] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.27.132.244 (local), world_size=2, local_world_size=2
|
||||
(Worker pid=2661557) INFO 07-12 15:17:04 [parallel_state.py:1588] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:48921 backend=nccl
|
||||
(Worker pid=2661558) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:48921 backend=nccl
|
||||
(Worker pid=2661557) INFO 07-12 15:17:06 [pynccl.py:113] vLLM is using nccl==2.28.9
|
||||
(Worker pid=2661557) INFO 07-12 15:17:08 [cuda_communicator.py:245] Using ['CUSTOM', 'SYMM_MEM', 'PYNCCL'] all-reduce backends (in dispatch order) for group 'tp:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661557) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['PYNCCL'] all-reduce backends (in dispatch order) for group 'ep:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661557) INFO 07-12 15:17:09 [parallel_state.py:1923] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(Worker pid=2661557) INFO 07-12 15:17:09 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:09 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1275.69 GiB.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:10 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:08, 1.71it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:09, 1.55it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:01<00:08, 1.52it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:02<00:08, 1.48it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:03<00:07, 1.47it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:04<00:06, 1.45it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:04<00:06, 1.44it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:05<00:05, 1.45it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:06<00:04, 1.44it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:06<00:04, 1.44it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:07<00:03, 1.43it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:08<00:02, 1.43it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:08<00:02, 1.42it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:09<00:01, 1.42it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:10<00:00, 1.44it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.87it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.52it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:21 [default_loader.py:430] Loading weights took 10.50 seconds
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:21 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:21 [gpu_model_runner.py:5259] Model loading took 28.46 GiB memory and 10.959169 seconds
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:31 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/71caca9a09/rank_0_0/backbone for vLLM's torch.compile
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:31 [backends.py:1148] Dynamo bytecode transform time: 9.70 s
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:31 [flashinfer_all_reduce.py:112] Auto-selected flashinfer allreduce backend: trtllm
|
||||
(Worker_TP0 pid=2661557) /tmp/wjh-opprof-phase2-dash0-20260711/.venv/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
||||
(Worker_TP0 pid=2661557) return func(*args, **kwargs)
|
||||
[rank0]:[W712 15:17:32.191117064 ProcessGroupNCCL.cpp:5188] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:32 [flashinfer_all_reduce.py:152] Initialized FlashInfer Allreduce norm fusion workspace with backend=trtllm
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:17:56 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:03 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 30.28 s
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:18 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/b1870a24d902681201272cd6a2355f94635aa2439d53d1b45755c3823de597b6/rank_0_0/model
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:18 [monitor.py:53] torch.compile took 56.43 s in total
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:18 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:20 [monitor.py:81] Initial profiling/warmup run took 2.40 s
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:22 [shm_broadcast.py:705] No available shared memory broadcast block found in 60 seconds. This typically happens when some processes are hanging or doing some time-consuming work (e.g. compilation, weight/kv cache quantization).
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=7 (largest=32), FULL=5 (largest=16)
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.08 GiB total
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:35 [gpu_worker.py:508] Available KV cache memory: 57.35 GiB
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:35 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9191 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9209. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:35 [kv_cache_utils.py:2146] GPU KV cache size: 1,252,864 tokens
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:35 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 30.59x
|
||||
(Worker_TP1 pid=2661558) 2026-07-12 15:18:35,109 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(Worker_TP0 pid=2661557) 2026-07-12 15:18:35,114 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP1 pid=2661558) 2026-07-12 15:18:35,169 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661557) 2026-07-12 15:18:35,170 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661557)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 14%|█▍ | 1/7 [00:00<00:00, 7.87it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 29%|██▊ | 2/7 [00:00<00:00, 8.09it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 43%|████▎ | 3/7 [00:00<00:00, 8.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 4/7 [00:00<00:00, 8.19it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 71%|███████▏ | 5/7 [00:02<00:01, 1.38it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▌ | 6/7 [00:03<00:00, 1.17it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:04<00:00, 1.09it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 7/7 [00:04<00:00, 1.58it/s]
|
||||
(Worker_TP0 pid=2661557)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 20%|██ | 1/5 [00:00<00:00, 8.34it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 40%|████ | 2/5 [00:00<00:00, 8.87it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 60%|██████ | 3/5 [00:00<00:00, 9.15it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 80%|████████ | 4/5 [00:00<00:00, 9.30it/s](Worker_TP1 pid=2661558) INFO 07-12 15:18:40 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 9.17it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 5/5 [00:00<00:00, 9.09it/s]
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:40 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:41 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.01 GiB (14.0%).
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:41 [gpu_model_runner.py:6660] Graph capturing finished in 6 secs, took 0.10 GiB
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:41 [gpu_worker.py:667] CUDA graph pool memory: 0.1 GiB (actual), 0.08 GiB (estimated), difference: 0.01 GiB (14.0%).
|
||||
(Worker_TP1 pid=2661558) INFO 07-12 15:18:41 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(Worker_TP0 pid=2661557) INFO 07-12 15:18:41 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:41 [core.py:337] init engine (profile, create kv cache, warmup model) took 79.74 s (compilation: 56.45 s)
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:42 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns16/opprof/opprof-v1-dp0-pid2661159-1783869522696890561.jsonl
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:42 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:42 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2661159) INFO 07-12 15:18:43 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:43 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2660455) WARNING 07-12 15:18:43 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8511
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2660455) INFO 07-12 15:18:44 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2660455) INFO: Started server process [2660455]
|
||||
(APIServer pid=2660455) INFO: Waiting for application startup.
|
||||
(APIServer pid=2660455) INFO: Application startup complete.
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:51420 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661557) WARNING 07-12 15:18:54 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57856 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57896 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57908 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661557) WARNING 07-12 15:18:56 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:57942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:04 [loggers.py:273] Engine 000: Avg prompt throughput: 5201.1 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.4%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:49404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52138 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52142 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52170 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52186 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:14 [loggers.py:273] Engine 000: Avg prompt throughput: 25.9 tokens/s, Avg generation throughput: 193.9 tokens/s, Running: 5 reqs, Waiting: 1 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 47.6%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52402 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52526 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:52532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48448 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48450 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48460 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48492 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48532 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48540 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:24 [loggers.py:273] Engine 000: Avg prompt throughput: 8173.6 tokens/s, Avg generation throughput: 281.9 tokens/s, Running: 16 reqs, Waiting: 27 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 32.6%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48558 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48560 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48572 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48616 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48626 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:48662 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:34 [loggers.py:273] Engine 000: Avg prompt throughput: 5964.6 tokens/s, Avg generation throughput: 336.0 tokens/s, Running: 11 reqs, Waiting: 12 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 27.1%
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:44 [loggers.py:273] Engine 000: Avg prompt throughput: 3010.9 tokens/s, Avg generation throughput: 276.2 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 25.0%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41702 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:41716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44524 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44594 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44598 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44634 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:19:54 [loggers.py:273] Engine 000: Avg prompt throughput: 21.7 tokens/s, Avg generation throughput: 239.7 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 38.8%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44642 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44650 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44674 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44708 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44712 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44720 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44758 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:44888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34408 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34478 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34562 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:04 [loggers.py:273] Engine 000: Avg prompt throughput: 46.9 tokens/s, Avg generation throughput: 670.8 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 56.6%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34582 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34586 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34602 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34636 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34678 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34694 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34696 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34710 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33600 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33608 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33618 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33628 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33652 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33668 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33682 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33688 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33700 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33726 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33730 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:14 [loggers.py:273] Engine 000: Avg prompt throughput: 8386.5 tokens/s, Avg generation throughput: 478.4 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 50.9%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33778 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33862 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33892 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:33998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58288 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58344 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:24 [loggers.py:273] Engine 000: Avg prompt throughput: 12462.8 tokens/s, Avg generation throughput: 461.2 tokens/s, Running: 16 reqs, Waiting: 1 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 44.1%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58384 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58394 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58420 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58434 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58504 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58506 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58520 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58536 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58548 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58552 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58566 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58610 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58112 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58138 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58172 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:34 [loggers.py:273] Engine 000: Avg prompt throughput: 10574.6 tokens/s, Avg generation throughput: 559.7 tokens/s, Running: 16 reqs, Waiting: 8 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 39.5%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58308 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58354 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58378 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58436 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58458 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58472 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58496 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58508 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:58516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34152 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34182 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34214 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO 07-12 15:20:44 [loggers.py:273] Engine 000: Avg prompt throughput: 14020.4 tokens/s, Avg generation throughput: 680.9 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 35.0%
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34280 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34304 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34310 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660455) INFO: 127.0.0.1:34336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp2_mns16/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns16/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns16", "anchor": 0.49609375, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp2_mns16/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp2_mns16/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 5.344845622,
|
||||
"end_mono_ns": 201112093538719,
|
||||
"end_wall_ns": 1783869539219965703,
|
||||
"start_mono_ns": 201106748693097,
|
||||
"start_wall_ns": 1783869533875120325
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 16,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "818d316f68acc6502ed33a877bf88c2ba9b57a1ff6d6780621db0d30eddf3494",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3.4938000000000105,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 7,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "2eb3b78b31efc9354c64c8caa95f08353330553d1ab745a8e0ceab038a3a3d5e",
|
||||
"request_id_order_sha256": "0009512d07ec4c5bb2dd1283c767967a6dfd5bc4c7b7d65fb26169d39a98832e"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 32.09012237012636,
|
||||
"min": 9.460583771716466,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 1483.7540120061021,
|
||||
"min": 370.18870600149967,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75, "kind": "anchor", "pass_rate": 0.4117647058823529, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 221,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 38.257993674,
|
||||
"end_mono_ns": 201200786085530,
|
||||
"end_wall_ns": 1783869627912512603,
|
||||
"start_mono_ns": 201162528091856,
|
||||
"start_wall_ns": 1783869589654519727
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 391,
|
||||
"pass_rate": 0.4117647058823529,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "d3ed714128e985d0e0a0ca6b3767210c181114ab6694fb066a60aecf5afa9f9a",
|
||||
"arrival_s": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"count": 391,
|
||||
"long_gt4096": 140,
|
||||
"offered_req_s": 6.516666666666667,
|
||||
"offered_req_s_per_gpu": 3.2583333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 361,
|
||||
"finite_n": 391,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"raw_length_order_sha256": "8152bbaf9f27e84e6fbd6359174d17de4b9c96ea8b8c58e3760290be3469b2a9",
|
||||
"request_id_order_sha256": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c"
|
||||
},
|
||||
"slo_pass_count": 161,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 221,
|
||||
"finite_n": 221,
|
||||
"max": 59.45439488186105,
|
||||
"min": 6.009525047224675,
|
||||
"missing_n": 170,
|
||||
"n": 391
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 221,
|
||||
"finite_n": 221,
|
||||
"max": 3492.7126489928924,
|
||||
"min": 39.199416001793,
|
||||
"missing_n": 170,
|
||||
"n": 391
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75390625, "kind": "anchor", "pass_rate": 0.027918781725888325, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75390625,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 110,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 27.144585398,
|
||||
"end_mono_ns": 201149546363473,
|
||||
"end_wall_ns": 1783869576672790977,
|
||||
"start_mono_ns": 201122401778075,
|
||||
"start_wall_ns": 1783869549528204982
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 394,
|
||||
"pass_rate": 0.027918781725888325,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "60fc43518e79fd46abe59afde67663a762ca17e817202fa52803040ac9056796",
|
||||
"arrival_s": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"count": 394,
|
||||
"long_gt4096": 142,
|
||||
"offered_req_s": 6.566666666666666,
|
||||
"offered_req_s_per_gpu": 3.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 364,
|
||||
"finite_n": 394,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"raw_length_order_sha256": "4a9cc03e351cce8c558b30732418c878fc3b6d24e64bef6717ce7c87c384384f",
|
||||
"request_id_order_sha256": "2677a6a12ff0e104d6246dd121d0667ff4fe3d25f8d68699547906420246ebdd"
|
||||
},
|
||||
"slo_pass_count": 11,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 110,
|
||||
"finite_n": 110,
|
||||
"max": 85.24905312595537,
|
||||
"min": 5.966831590589796,
|
||||
"missing_n": 284,
|
||||
"n": 394
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 110,
|
||||
"finite_n": 110,
|
||||
"max": 8747.10591501207,
|
||||
"min": 43.805207998957485,
|
||||
"missing_n": 284,
|
||||
"n": 394
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp2_mns32/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp2_mns32/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns32",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 17712,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/opprof/opprof-v1-dp0-pid2661056-1783869524249984543.jsonl"
|
||||
}
|
||||
6
runs/opprof-phase6/phase6/cells/tp2_mns32/commands.log
Normal file
6
runs/opprof-phase6/phase6/cells/tp2_mns32/commands.log
Normal file
@@ -0,0 +1,6 @@
|
||||
SERVER taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8512 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 32 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/warmup
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/anchor-0.75390625
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/anchor-0.75
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75390625 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/confirm-1-anchor-0.75390625
|
||||
CLIENT taskset -c 80-99,100-119 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns32 --anchor 0.75 --tp 2 --mns 32 --base-url http://127.0.0.1:8512 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns32/confirm-2-anchor-0.75
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75390625, "kind": "anchor", "pass_rate": 0.9568527918781726, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75390625,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 394,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 64.605127417,
|
||||
"end_mono_ns": 201294724925354,
|
||||
"end_wall_ns": 1783869721851353202,
|
||||
"start_mono_ns": 201230119797937,
|
||||
"start_wall_ns": 1783869657246225290
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 394,
|
||||
"pass_rate": 0.9568527918781726,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "60fc43518e79fd46abe59afde67663a762ca17e817202fa52803040ac9056796",
|
||||
"arrival_s": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"count": 394,
|
||||
"long_gt4096": 142,
|
||||
"offered_req_s": 6.566666666666666,
|
||||
"offered_req_s_per_gpu": 3.283333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 364,
|
||||
"finite_n": 394,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"raw_length_order_sha256": "4a9cc03e351cce8c558b30732418c878fc3b6d24e64bef6717ce7c87c384384f",
|
||||
"request_id_order_sha256": "2677a6a12ff0e104d6246dd121d0667ff4fe3d25f8d68699547906420246ebdd"
|
||||
},
|
||||
"slo_pass_count": 377,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 50.88291912592625,
|
||||
"min": 5.975462165367826,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 394,
|
||||
"finite_n": 394,
|
||||
"max": 3372.0373090181965,
|
||||
"min": 39.65454100398347,
|
||||
"missing_n": 0,
|
||||
"n": 394
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 391,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 60.782576125,
|
||||
"end_mono_ns": 201361364171726,
|
||||
"end_wall_ns": 1783869788490598727,
|
||||
"start_mono_ns": 201300581595601,
|
||||
"start_wall_ns": 1783869727708022566
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 32,
|
||||
"observed_count": 391,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "d3ed714128e985d0e0a0ca6b3767210c181114ab6694fb066a60aecf5afa9f9a",
|
||||
"arrival_s": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"count": 391,
|
||||
"long_gt4096": 140,
|
||||
"offered_req_s": 6.516666666666667,
|
||||
"offered_req_s_per_gpu": 3.2583333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 361,
|
||||
"finite_n": 391,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"raw_length_order_sha256": "8152bbaf9f27e84e6fbd6359174d17de4b9c96ea8b8c58e3760290be3469b2a9",
|
||||
"request_id_order_sha256": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c"
|
||||
},
|
||||
"slo_pass_count": 391,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 9.731004889788634,
|
||||
"min": 5.9432724644759976,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 81.50044700596482,
|
||||
"min": 39.63042498799041,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661056-1783869524249984543.jsonl","encoded_records":17712,"written_records":17712,"dropped_records":0,"last_step_index":17711,"checkpoint_wall_ns":1783869804704938995,"flush_interval_seconds":1.0,"final":true}
|
||||
1313
runs/opprof-phase6/phase6/cells/tp2_mns32/server.log
Normal file
1313
runs/opprof-phase6/phase6/cells/tp2_mns32/server.log
Normal file
File diff suppressed because it is too large
Load Diff
1
runs/opprof-phase6/phase6/cells/tp2_mns32/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns32/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns32", "anchor": 0.75390625, "kind": "warmup", "pass_rate": 0.75, "feasible": false}
|
||||
74
runs/opprof-phase6/phase6/cells/tp2_mns32/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp2_mns32/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75390625,
|
||||
"cell": "tp2_mns32",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 7.481165294,
|
||||
"end_mono_ns": 201114243480033,
|
||||
"end_wall_ns": 1783869541369906988,
|
||||
"start_mono_ns": 201106762314739,
|
||||
"start_wall_ns": 1783869533888742060
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 32,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 0.75,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "440dc3774dd5b89ed800eee12e44a20f62b0ee9b66094d8e05fa343fb2ba3dd7",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 2.4043000000000125,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 5,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7233.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "cb99e1339d84161d25fbc6a3c6ed983f34c0276e7712a62c176a8b92b8cb7b93",
|
||||
"request_id_order_sha256": "9d18ed5ddc01d283277e84d107b444833f6ab2838675a8c7ffaedb7a0c1d50cf"
|
||||
},
|
||||
"slo_pass_count": 12,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 48.713730740283914,
|
||||
"min": 9.619937283601098,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3962.258840998402,
|
||||
"min": 814.5189289934933,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
1
runs/opprof-phase6/phase6/cells/tp2_mns64/anchor-0.5.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns64/anchor-0.5.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns64", "anchor": 0.5, "kind": "anchor", "pass_rate": 0.4601449275362319, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.5,
|
||||
"cell": "tp2_mns64",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 142,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 32.625174607,
|
||||
"end_mono_ns": 201195153573715,
|
||||
"end_wall_ns": 1783869622280001349,
|
||||
"start_mono_ns": 201162528399108,
|
||||
"start_wall_ns": 1783869589654826388
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 276,
|
||||
"pass_rate": 0.4601449275362319,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "aeea4a8e54cc7de279804b8c353f748a480aedcbb21e249a90a60267daee8dff",
|
||||
"arrival_s": {
|
||||
"distinct_n": 276,
|
||||
"finite_n": 276,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"count": 276,
|
||||
"long_gt4096": 99,
|
||||
"offered_req_s": 4.6,
|
||||
"offered_req_s_per_gpu": 2.3,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 256,
|
||||
"finite_n": 276,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 276
|
||||
},
|
||||
"raw_length_order_sha256": "983df528039301bd41a6684fecea75cdcf8822d214094d6d35def7fc7740c2ba",
|
||||
"request_id_order_sha256": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15"
|
||||
},
|
||||
"slo_pass_count": 127,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 142,
|
||||
"finite_n": 142,
|
||||
"max": 65.60344734654475,
|
||||
"min": 4.993453378033802,
|
||||
"missing_n": 134,
|
||||
"n": 276
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 142,
|
||||
"finite_n": 142,
|
||||
"max": 1354.7493759833742,
|
||||
"min": 39.566546998685226,
|
||||
"missing_n": 134,
|
||||
"n": 276
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns64", "anchor": 0.75, "kind": "anchor", "pass_rate": 0.14066496163682865, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns64",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 129,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 26.447176041,
|
||||
"end_mono_ns": 201148848516622,
|
||||
"end_wall_ns": 1783869575974943780,
|
||||
"start_mono_ns": 201122401340581,
|
||||
"start_wall_ns": 1783869549527767999
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 64,
|
||||
"observed_count": 391,
|
||||
"pass_rate": 0.14066496163682865,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "d3ed714128e985d0e0a0ca6b3767210c181114ab6694fb066a60aecf5afa9f9a",
|
||||
"arrival_s": {
|
||||
"distinct_n": 391,
|
||||
"finite_n": 391,
|
||||
"max": 59.91880000000001,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"count": 391,
|
||||
"long_gt4096": 140,
|
||||
"offered_req_s": 6.516666666666667,
|
||||
"offered_req_s_per_gpu": 3.2583333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 361,
|
||||
"finite_n": 391,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 391
|
||||
},
|
||||
"raw_length_order_sha256": "8152bbaf9f27e84e6fbd6359174d17de4b9c96ea8b8c58e3760290be3469b2a9",
|
||||
"request_id_order_sha256": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c"
|
||||
},
|
||||
"slo_pass_count": 55,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 129,
|
||||
"max": 109.8500202991083,
|
||||
"min": 5.983953999894429,
|
||||
"missing_n": 262,
|
||||
"n": 391
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 129,
|
||||
"finite_n": 129,
|
||||
"max": 2141.4210819930304,
|
||||
"min": 43.8292009930592,
|
||||
"missing_n": 262,
|
||||
"n": 391
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp2_mns64/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp2_mns64/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns64",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 4537,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/opprof/opprof-v1-dp0-pid2661205-1783869525995600880.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp2_mns64/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp2_mns64/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8513 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 64 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns64 --anchor 0.75 --tp 2 --mns 64 --base-url http://127.0.0.1:8513 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/warmup
|
||||
CLIENT taskset -c 120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns64 --anchor 0.75 --tp 2 --mns 64 --base-url http://127.0.0.1:8513 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/anchor-0.75
|
||||
CLIENT taskset -c 120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns64 --anchor 0.5 --tp 2 --mns 64 --base-url http://127.0.0.1:8513 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/anchor-0.5
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661205-1783869525995600880.jsonl","encoded_records":4537,"written_records":4537,"dropped_records":0,"last_step_index":4536,"checkpoint_wall_ns":1783869810430748669,"flush_interval_seconds":1.0,"final":true}
|
||||
453
runs/opprof-phase6/phase6/cells/tp2_mns64/server.log
Normal file
453
runs/opprof-phase6/phase6/cells/tp2_mns64/server.log
Normal file
@@ -0,0 +1,453 @@
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8513, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'tensor_parallel_size': 2, 'max_num_batched_tokens': 8192, 'max_num_seqs': 64, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:40 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2660457) INFO 07-12 15:16:44 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(EngineCore pid=2661205) INFO 07-12 15:16:54 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': True, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2661205) WARNING 07-12 15:16:54 [multiproc_executor.py:1063] Reducing Torch parallelism from 40 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
(EngineCore pid=2661205) INFO 07-12 15:16:54 [multiproc_executor.py:140] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.27.132.244 (local), world_size=2, local_world_size=2
|
||||
(Worker pid=2661569) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:41317 backend=nccl
|
||||
(Worker pid=2661570) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:41317 backend=nccl
|
||||
(Worker pid=2661569) INFO 07-12 15:17:06 [pynccl.py:113] vLLM is using nccl==2.28.9
|
||||
(Worker pid=2661569) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['CUSTOM', 'SYMM_MEM', 'PYNCCL'] all-reduce backends (in dispatch order) for group 'tp:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661569) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['PYNCCL'] all-reduce backends (in dispatch order) for group 'ep:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661569) INFO 07-12 15:17:09 [parallel_state.py:1923] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(Worker pid=2661569) INFO 07-12 15:17:10 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:10 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:10 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:10 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:10 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:11 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1275.25 GiB.
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:11 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:09, 1.58it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:09, 1.40it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:02<00:09, 1.36it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:02<00:08, 1.36it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:03<00:08, 1.35it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:04<00:07, 1.36it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:05<00:06, 1.36it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:05<00:05, 1.35it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:06<00:05, 1.33it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:07<00:04, 1.33it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:08<00:03, 1.32it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:08<00:03, 1.30it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:09<00:02, 1.32it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:10<00:01, 1.33it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:11<00:00, 1.34it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:11<00:00, 1.74it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:11<00:00, 1.41it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:22 [default_loader.py:430] Loading weights took 11.35 seconds
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:22 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:23 [gpu_model_runner.py:5259] Model loading took 28.46 GiB memory and 11.752905 seconds
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:32 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/7a9372c548/rank_0_0/backbone for vLLM's torch.compile
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:32 [backends.py:1148] Dynamo bytecode transform time: 9.56 s
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:32 [flashinfer_all_reduce.py:112] Auto-selected flashinfer allreduce backend: trtllm
|
||||
(Worker_TP0 pid=2661569) /tmp/wjh-opprof-phase2-dash0-20260711/.venv/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
||||
(Worker_TP0 pid=2661569) return func(*args, **kwargs)
|
||||
[rank0]:[W712 15:17:33.273268179 ProcessGroupNCCL.cpp:5188] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:33 [flashinfer_all_reduce.py:152] Initialized FlashInfer Allreduce norm fusion workspace with backend=trtllm
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:17:57 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:04 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 30.15 s
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:18 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/7c6e0ddc6dcbaf895fc62cc90d8a628d2dfd8c5b38e2d7747d4762b6a26b182b/rank_0_0/model
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:18 [monitor.py:53] torch.compile took 55.50 s in total
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:19 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:21 [monitor.py:81] Initial profiling/warmup run took 2.55 s
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:24 [shm_broadcast.py:705] No available shared memory broadcast block found in 60 seconds. This typically happens when some processes are hanging or doing some time-consuming work (e.g. compilation, weight/kv cache quantization).
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=19 (largest=128), FULL=11 (largest=64)
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=19 (largest=128), FULL=11 (largest=64)
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:32 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:32 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:33 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.17 GiB total
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:33 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.17 GiB total
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9182 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9218. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:34 [gpu_worker.py:508] Available KV cache memory: 57.27 GiB
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:34 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9182 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9218. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:34 [kv_cache_utils.py:2146] GPU KV cache size: 1,251,040 tokens
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:34 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 30.54x
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:34 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:34 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(Worker_TP1 pid=2661570) 2026-07-12 15:18:34,433 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP0 pid=2661569) 2026-07-12 15:18:34,434 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP1 pid=2661570) 2026-07-12 15:18:34,488 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661569) 2026-07-12 15:18:34,489 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661569)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:02, 8.64it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:00<00:01, 8.66it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:00<00:01, 8.61it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:00<00:01, 8.68it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:00<00:01, 8.60it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:00<00:01, 8.62it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:00<00:01, 8.61it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:00<00:01, 8.62it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:01<00:01, 8.67it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:01<00:01, 8.67it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:01<00:00, 8.59it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:02<00:03, 2.22it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:02<00:02, 2.84it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:03<00:02, 1.71it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:03<00:01, 2.24it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:03<00:01, 2.87it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:05<00:01, 1.37it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:06<00:00, 1.25it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 1.15it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 2.50it/s]
|
||||
(Worker_TP0 pid=2661569)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:01, 9.49it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:00, 9.41it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:00<00:00, 9.08it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:00<00:00, 9.02it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:00<00:00, 8.97it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:00<00:00, 8.15it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:00<00:00, 8.52it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:00<00:00, 8.81it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:01<00:00, 8.94it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:01<00:00, 9.03it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:01<00:00, 9.16it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:01<00:00, 8.94it/s]
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:43 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:43 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:44 [gpu_model_runner.py:6660] Graph capturing finished in 10 secs, took 0.20 GiB
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:44 [gpu_worker.py:667] CUDA graph pool memory: 0.2 GiB (actual), 0.17 GiB (estimated), difference: 0.03 GiB (14.9%).
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:44 [gpu_worker.py:667] CUDA graph pool memory: 0.2 GiB (actual), 0.17 GiB (estimated), difference: 0.03 GiB (14.9%).
|
||||
(Worker_TP0 pid=2661569) INFO 07-12 15:18:44 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(Worker_TP1 pid=2661570) INFO 07-12 15:18:44 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:44 [core.py:337] init engine (profile, create kv cache, warmup model) took 81.68 s (compilation: 55.67 s)
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:45 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns64/opprof/opprof-v1-dp0-pid2661205-1783869525995600880.jsonl
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:46 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:46 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2661205) INFO 07-12 15:18:47 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2660457) WARNING 07-12 15:18:47 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8513
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:47 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2660457) INFO: Started server process [2660457]
|
||||
(APIServer pid=2660457) INFO: Waiting for application startup.
|
||||
(APIServer pid=2660457) INFO: Application startup complete.
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:57528 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55578 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55590 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55592 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55620 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661569) WARNING 07-12 15:18:54 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55624 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55632 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55654 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55676 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55686 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:55706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661569) WARNING 07-12 15:18:56 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660457) INFO 07-12 15:18:57 [loggers.py:273] Engine 000: Avg prompt throughput: 3062.3 tokens/s, Avg generation throughput: 3.4 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 7.2%
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:07 [loggers.py:273] Engine 000: Avg prompt throughput: 922.8 tokens/s, Avg generation throughput: 200.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 6.5%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50272 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50296 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:50306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47640 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47658 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47686 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47698 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47706 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47716 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47724 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47750 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47764 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47786 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47818 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47828 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47866 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47872 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47898 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47942 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:47984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48018 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:17 [loggers.py:273] Engine 000: Avg prompt throughput: 7057.2 tokens/s, Avg generation throughput: 208.8 tokens/s, Running: 28 reqs, Waiting: 8 reqs, GPU KV cache usage: 6.9%, Prefix cache hit rate: 30.7%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48040 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48088 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:48116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43188 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43242 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43250 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43256 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43314 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:27 [loggers.py:273] Engine 000: Avg prompt throughput: 13039.6 tokens/s, Avg generation throughput: 707.1 tokens/s, Running: 63 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.9%, Prefix cache hit rate: 21.3%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43322 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43348 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43380 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43386 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43412 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43426 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43456 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:43480 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40292 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40362 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40370 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40382 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40442 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40464 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40470 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40486 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40500 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40510 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40512 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:40530 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:37 [loggers.py:273] Engine 000: Avg prompt throughput: 13455.8 tokens/s, Avg generation throughput: 735.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.6%
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:47 [loggers.py:273] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 16.6%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:33978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54484 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54488 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54502 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54516 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54518 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54522 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54534 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54544 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54556 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54568 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54580 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54596 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54606 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54622 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54646 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54660 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54672 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54680 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54692 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54704 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54718 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54734 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54756 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54760 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54774 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54810 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:19:57 [loggers.py:273] Engine 000: Avg prompt throughput: 36.2 tokens/s, Avg generation throughput: 441.2 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 33.0%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54826 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54848 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54850 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54878 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54886 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:54936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49096 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49126 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49136 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49144 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49168 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49194 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49228 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49270 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49326 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49346 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49352 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49388 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49410 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49414 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO 07-12 15:20:07 [loggers.py:273] Engine 000: Avg prompt throughput: 45.9 tokens/s, Avg generation throughput: 672.0 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 46.1%
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49422 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49424 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49440 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49462 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49468 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49490 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:49494 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53746 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53754 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53766 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53814 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53830 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53840 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53846 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53868 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660457) INFO: 127.0.0.1:53884 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp2_mns64/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns64/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns64", "anchor": 0.75, "kind": "warmup", "pass_rate": 0.75, "feasible": false}
|
||||
74
runs/opprof-phase6/phase6/cells/tp2_mns64/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp2_mns64/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.75,
|
||||
"cell": "tp2_mns64",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 7.415686836,
|
||||
"end_mono_ns": 201114188199246,
|
||||
"end_wall_ns": 1783869541314626167,
|
||||
"start_mono_ns": 201106772512410,
|
||||
"start_wall_ns": 1783869533898939505
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 64,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 0.75,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "97f6ded38afa2a0fa9729d0553429b3df62e3d4e99710f4c19a706b1eba8fdaa",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 2.723000000000002,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 5,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7233.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "76f4ce7bc309a1108d5d4b24ba24b374c14483a3dd35cd5648fbeafca60221ba",
|
||||
"request_id_order_sha256": "b38587f14f8c188d131b4c5c8411951ec571999fb2ac6be6517f1ad952208852"
|
||||
},
|
||||
"slo_pass_count": 12,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 48.44081654317868,
|
||||
"min": 9.34166979529587,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3932.37450599554,
|
||||
"min": 783.6844140256289,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns8", "anchor": 0.4921875, "kind": "anchor", "pass_rate": 0.6914498141263941, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.4921875,
|
||||
"cell": "tp2_mns8",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 216,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 51.56724132,
|
||||
"end_mono_ns": 201214250324754,
|
||||
"end_wall_ns": 1783869641376751823,
|
||||
"start_mono_ns": 201162683083434,
|
||||
"start_wall_ns": 1783869589809510519
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 269,
|
||||
"pass_rate": 0.6914498141263941,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "ffbc7b8dd324c8e745654110a2412b03ddade891beb2bfe022ffde6502b0184b",
|
||||
"arrival_s": {
|
||||
"distinct_n": 269,
|
||||
"finite_n": 269,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"count": 269,
|
||||
"long_gt4096": 95,
|
||||
"offered_req_s": 4.483333333333333,
|
||||
"offered_req_s_per_gpu": 2.2416666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 250,
|
||||
"finite_n": 269,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 269
|
||||
},
|
||||
"raw_length_order_sha256": "bc951350376a487d34d2def78fd32a2286e23bf8a7f481ff578009e12e3f1fb8",
|
||||
"request_id_order_sha256": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c"
|
||||
},
|
||||
"slo_pass_count": 186,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 216,
|
||||
"finite_n": 216,
|
||||
"max": 22.936211125952685,
|
||||
"min": 5.354799519524008,
|
||||
"missing_n": 53,
|
||||
"n": 269
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 216,
|
||||
"finite_n": 216,
|
||||
"max": 3692.623541021021,
|
||||
"min": 40.143018995877355,
|
||||
"missing_n": 53,
|
||||
"n": 269
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns8", "anchor": 0.49609375, "kind": "anchor", "pass_rate": 0.09157509157509157, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns8",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 87,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 33.978401963,
|
||||
"end_mono_ns": 201156432397399,
|
||||
"end_wall_ns": 1783869583558824468,
|
||||
"start_mono_ns": 201122453995436,
|
||||
"start_wall_ns": 1783869549580422810
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 273,
|
||||
"pass_rate": 0.09157509157509157,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "41168f4bd02f3ef4e83b7440a3dab7458f08df9c2bbd4452fad076b4a2e0eaed",
|
||||
"arrival_s": {
|
||||
"distinct_n": 273,
|
||||
"finite_n": 273,
|
||||
"max": 59.78950000000005,
|
||||
"min": 0.008300000000008368,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"count": 273,
|
||||
"long_gt4096": 97,
|
||||
"offered_req_s": 4.55,
|
||||
"offered_req_s_per_gpu": 2.275,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 253,
|
||||
"finite_n": 273,
|
||||
"max": 8149.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 273
|
||||
},
|
||||
"raw_length_order_sha256": "9e38ed1766e3933dc05a7ced5d0a73fed47ce769800f0d2c10014af0ef823f28",
|
||||
"request_id_order_sha256": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80"
|
||||
},
|
||||
"slo_pass_count": 25,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 87,
|
||||
"finite_n": 87,
|
||||
"max": 38.90758789764116,
|
||||
"min": 6.895991826913957,
|
||||
"missing_n": 186,
|
||||
"n": 273
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 87,
|
||||
"finite_n": 87,
|
||||
"max": 16124.392641999293,
|
||||
"min": 52.06192599143833,
|
||||
"missing_n": 186,
|
||||
"n": 273
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp2_mns8/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp2_mns8/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp2_mns8",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 6687,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/opprof/opprof-v1-dp0-pid2661256-1783869521184806299.jsonl"
|
||||
}
|
||||
4
runs/opprof-phase6/phase6/cells/tp2_mns8/commands.log
Normal file
4
runs/opprof-phase6/phase6/cells/tp2_mns8/commands.log
Normal file
@@ -0,0 +1,4 @@
|
||||
SERVER taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8510 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 8 --tensor-parallel-size 2 --shutdown-timeout 120
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.49609375 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/warmup
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.49609375 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/anchor-0.49609375
|
||||
CLIENT taskset -c 0-19,20-39 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-primary.json --cell tp2_mns8 --anchor 0.4921875 --tp 2 --mns 8 --base-url http://127.0.0.1:8510 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/anchor-0.4921875
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2661256-1783869521184806299.jsonl","encoded_records":6687,"written_records":6687,"dropped_records":0,"last_step_index":6686,"checkpoint_wall_ns":1783869792959224277,"flush_interval_seconds":1.0,"final":true}
|
||||
486
runs/opprof-phase6/phase6/cells/tp2_mns8/server.log
Normal file
486
runs/opprof-phase6/phase6/cells/tp2_mns8/server.log
Normal file
@@ -0,0 +1,486 @@
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] █ █ █▄ ▄█
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] ▄▄ ▄█ █ █ █ ▀▄▀ █ version 0.24.1.dev3+g668cfb7e2
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] █▄█▀ █ █ █ █ model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339] ▀▀ ▀▀▀▀▀ ▀▀▀▀▀ ▀ ▀
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:339]
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [api_utils.py:273] non-default args: {'model_tag': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'host': '127.0.0.1', 'port': 8510, 'model': '/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', 'served_model_name': ['qwen3-30b-a3b-community'], 'tensor_parallel_size': 2, 'max_num_batched_tokens': 8192, 'max_num_seqs': 8, 'shutdown_timeout': 120}
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [model.py:598] Resolved architecture: Qwen3MoeForCausalLM
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [model.py:1725] Using max model len 40960
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [scheduler.py:252] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:40 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(APIServer pid=2660454) INFO 07-12 15:16:45 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(EngineCore pid=2661256) INFO 07-12 15:16:55 [core.py:114] Initializing a V1 LLM engine (v0.24.1.dev3+g668cfb7e2) with config: model='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', speculative_config=None, tokenizer='/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_verbose=False), seed=0, served_model_name=qwen3-30b-a3b-community, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::qwen_gdn_attention_core', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::kda_attention', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'size_asserts': False, 'alignment_asserts': False, 'scalar_asserts': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': True, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False}, 'max_cudagraph_capture_size': 16, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, moe_backend='auto', linear_backend='auto')
|
||||
(EngineCore pid=2661256) WARNING 07-12 15:16:55 [multiproc_executor.py:1063] Reducing Torch parallelism from 40 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
(EngineCore pid=2661256) INFO 07-12 15:16:55 [multiproc_executor.py:140] DP group leader: node_rank=0, node_rank_within_dp=0, master_addr=127.0.0.1, mq_connect_ip=172.27.132.244 (local), world_size=2, local_world_size=2
|
||||
(Worker pid=2661580) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:42171 backend=nccl
|
||||
(Worker pid=2661581) INFO 07-12 15:17:05 [parallel_state.py:1588] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:42171 backend=nccl
|
||||
(Worker pid=2661580) INFO 07-12 15:17:06 [pynccl.py:113] vLLM is using nccl==2.28.9
|
||||
(Worker pid=2661580) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['CUSTOM', 'SYMM_MEM', 'PYNCCL'] all-reduce backends (in dispatch order) for group 'tp:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661580) INFO 07-12 15:17:09 [cuda_communicator.py:245] Using ['PYNCCL'] all-reduce backends (in dispatch order) for group 'ep:0' out of potential backends: ['NCCL_SYMM_MEM', 'QUICK_REDUCE', 'FLASHINFER', 'CUSTOM', 'SYMM_MEM', 'PYNCCL'].
|
||||
(Worker pid=2661580) INFO 07-12 15:17:09 [parallel_state.py:1923] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0, EPLB rank N/A
|
||||
(Worker pid=2661580) INFO 07-12 15:17:10 [topk_topp_sampler.py:55] Using FlashInfer for top-p & top-k sampling.
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [gpu_model_runner.py:5164] Starting to load model /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B...
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [cuda.py:480] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [flash_attn.py:670] Using FlashAttention version 3
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [unquantized.py:247] Using TRITON Unquantized MoE backend out of potential backends: ['TRITON', 'BATCHED_TRITON', 'FlashInfer TRTLLM', 'FlashInfer CUTLASS'].
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [weight_utils.py:849] Filesystem type for checkpoints: FUSE.ALIYUN-ALINAS-EFC. Checkpoint size: 56.87 GiB. Available RAM: 1275.26 GiB.
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:10 [weight_utils.py:872] Auto-prefetch is disabled because the filesystem (FUSE.ALIYUN-ALINAS-EFC) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/16 [00:00<?, ?it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 6% Completed | 1/16 [00:00<00:09, 1.61it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 12% Completed | 2/16 [00:01<00:09, 1.53it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 19% Completed | 3/16 [00:01<00:08, 1.51it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 4/16 [00:02<00:08, 1.47it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 31% Completed | 5/16 [00:03<00:07, 1.45it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 38% Completed | 6/16 [00:04<00:06, 1.43it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 44% Completed | 7/16 [00:04<00:06, 1.44it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 8/16 [00:05<00:05, 1.45it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 56% Completed | 9/16 [00:06<00:04, 1.44it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 62% Completed | 10/16 [00:06<00:04, 1.44it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 69% Completed | 11/16 [00:07<00:03, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 12/16 [00:08<00:02, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 81% Completed | 13/16 [00:08<00:02, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 88% Completed | 14/16 [00:09<00:01, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 94% Completed | 15/16 [00:10<00:00, 1.46it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.89it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 16/16 [00:10<00:00, 1.53it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:21 [default_loader.py:430] Loading weights took 10.45 seconds
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:21 [unquantized.py:312] Using MoEPrepareAndFinalizeNoDPEPModular
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:22 [gpu_model_runner.py:5259] Model loading took 28.46 GiB memory and 10.837326 seconds
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [backends.py:1089] Using cache directory: /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/0c37cd99d7/rank_0_0/backbone for vLLM's torch.compile
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [backends.py:1148] Dynamo bytecode transform time: 9.74 s
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:32 [flashinfer_all_reduce.py:112] Auto-selected flashinfer allreduce backend: trtllm
|
||||
(Worker_TP0 pid=2661580) /tmp/wjh-opprof-phase2-dash0-20260711/.venv/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
|
||||
(Worker_TP0 pid=2661580) return func(*args, **kwargs)
|
||||
[rank0]:[W712 15:17:32.753967979 ProcessGroupNCCL.cpp:5188] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group()
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:33 [flashinfer_all_reduce.py:152] Initialized FlashInfer Allreduce norm fusion workspace with backend=trtllm
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:17:56 [backends.py:378] Cache the graph of compile range (1, 8192) for later use
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:03 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 30.05 s
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [decorators.py:708] saved AOT compiled function to /home/admin/cpfs/wjh/.cache/vllm/torch_compile_cache/torch_aot_compile/7db77e4784ad115c73ccf5b7472a39190ea4c85a91f415675a26d40f2c108d60/rank_0_0/model
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [monitor.py:53] torch.compile took 56.36 s in total
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:18 [fused_moe.py:1058] Using configuration from /home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=NVIDIA_H20.json for MoE layer.
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:21 [monitor.py:81] Initial profiling/warmup run took 2.52 s
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:23 [shm_broadcast.py:705] No available shared memory broadcast block found in 60 seconds. This typically happens when some processes are hanging or doing some time-consuming work (e.g. compilation, weight/kv cache quantization).
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:28 [gpu_model_runner.py:6487] Profiling CUDA graph memory: PIECEWISE=5 (largest=16), FULL=4 (largest=8)
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:33 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:34 [gpu_model_runner.py:6592] Estimated CUDA graph memory: 0.07 GiB total
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [gpu_worker.py:508] Available KV cache memory: 57.37 GiB
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:35 [gpu_worker.py:523] CUDA graph memory profiling is enabled (default since v0.21.0). The current --gpu-memory-utilization=0.9200 is equivalent to --gpu-memory-utilization=0.9193 without CUDA graph memory profiling. To maintain the same effective KV cache size as before, increase --gpu-memory-utilization to 0.9207. To disable, set VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:35 [kv_cache_utils.py:2146] GPU KV cache size: 1,253,168 tokens
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:35 [kv_cache_utils.py:2147] Maximum concurrency for 40,960 tokens per request: 30.59x
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [deep_gemm.py:175] deep_gemm not found in site-packages, trying vendored vllm.third_party.deep_gemm
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:35 [deep_gemm.py:202] DeepGEMM PDL enabled on vllm.third_party.deep_gemm.
|
||||
(Worker_TP0 pid=2661580) 2026-07-12 15:18:35,188 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP1 pid=2661581) 2026-07-12 15:18:35,200 - INFO - autotuner.py:622 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||||
(Worker_TP0 pid=2661580) 2026-07-12 15:18:35,244 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP1 pid=2661581) 2026-07-12 15:18:35,270 - INFO - autotuner.py:641 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||||
(Worker_TP0 pid=2661580)
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/5 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 20%|██ | 1/5 [00:00<00:00, 8.16it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 40%|████ | 2/5 [00:00<00:00, 8.50it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 60%|██████ | 3/5 [00:00<00:00, 8.72it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 4/5 [00:01<00:00, 1.89it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 1.37it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:02<00:00, 1.93it/s]
|
||||
(Worker_TP0 pid=2661580)
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/4 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 25%|██▌ | 1/4 [00:00<00:00, 8.58it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 50%|█████ | 2/4 [00:00<00:00, 9.08it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 75%|███████▌ | 3/4 [00:00<00:00, 9.18it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 9.44it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 4/4 [00:00<00:00, 9.28it/s]
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:38 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:38 [custom_all_reduce.py:213] Registering 0 cuda graph addresses
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:39 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (14.3%).
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [gpu_model_runner.py:6660] Graph capturing finished in 4 secs, took 0.08 GiB
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [gpu_worker.py:667] CUDA graph pool memory: 0.08 GiB (actual), 0.07 GiB (estimated), difference: 0.01 GiB (14.3%).
|
||||
(Worker_TP1 pid=2661581) INFO 07-12 15:18:39 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(Worker_TP0 pid=2661580) INFO 07-12 15:18:39 [jit_monitor.py:60] Kernel JIT monitor activated — Triton JIT compilations during inference will be logged as warnings.
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:39 [core.py:337] init engine (profile, create kv cache, warmup model) took 77.56 s (compilation: 56.36 s)
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:41 [scheduler.py:282] OpProf telemetry enabled: /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp2_mns8/opprof/opprof-v1-dp0-pid2661256-1783869521184806299.jsonl
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:41 [vllm.py:1006] Asynchronous scheduling is enabled.
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:41 [kernel.py:276] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(EngineCore pid=2661256) INFO 07-12 15:18:42 [compilation.py:310] Enabled custom fusions: allreduce_rms
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:42 [api_server.py:577] Supported tasks: ['generate']
|
||||
(APIServer pid=2660454) WARNING 07-12 15:18:42 [model.py:1477] Default vLLM sampling parameters have been overridden by the model's `generation_config.json`: `{'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}`. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:42 [hf.py:548] Detected the chat template content format to be 'string'. You can set `--chat-template-content-format` to override this.
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [api_server.py:581] Starting vLLM server on http://127.0.0.1:8510
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:37] Available routes are:
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /load, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /version, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /health, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /ping, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /ping, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/batch, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/messages/count_tokens, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /generative_scoring, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/render, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions/render, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/chat/completions/derender, Methods: POST
|
||||
(APIServer pid=2660454) INFO 07-12 15:18:43 [launcher.py:46] Route: /v1/completions/derender, Methods: POST
|
||||
(APIServer pid=2660454) INFO: Started server process [2660454]
|
||||
(APIServer pid=2660454) INFO: Waiting for application startup.
|
||||
(APIServer pid=2660454) INFO: Application startup complete.
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:34364 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42998 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661580) WARNING 07-12 15:18:54 [jit_monitor.py:106] Triton kernel JIT compilation during inference: _compute_slot_mapping_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43044 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43050 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43082 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43128 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(Worker_TP0 pid=2661580) WARNING 07-12 15:18:57 [jit_monitor.py:106] Triton kernel JIT compilation during inference: fused_moe_kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:43140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:03 [loggers.py:273] Engine 000: Avg prompt throughput: 5201.2 tokens/s, Avg generation throughput: 204.8 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 4.4%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52212 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41870 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41882 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41890 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41900 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41912 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41936 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41952 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:13 [loggers.py:273] Engine 000: Avg prompt throughput: 11.8 tokens/s, Avg generation throughput: 110.7 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.8%, Prefix cache hit rate: 50.0%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41968 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:41986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42008 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42016 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42080 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42094 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42110 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42146 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42148 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42164 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42210 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42230 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42246 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42252 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42260 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42262 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42276 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42298 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42160 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42174 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42190 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42192 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42208 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42218 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42224 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42240 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42264 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42268 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42296 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42312 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:23 [loggers.py:273] Engine 000: Avg prompt throughput: 5066.8 tokens/s, Avg generation throughput: 298.5 tokens/s, Running: 8 reqs, Waiting: 33 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 36.8%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42328 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42336 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42338 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42340 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42364 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42366 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42368 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42372 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42376 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42392 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42398 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42404 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42406 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42418 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42428 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42444 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:42452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:33 [loggers.py:273] Engine 000: Avg prompt throughput: 5845.7 tokens/s, Avg generation throughput: 315.7 tokens/s, Running: 7 reqs, Waiting: 26 reqs, GPU KV cache usage: 1.6%, Prefix cache hit rate: 29.2%
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:43 [loggers.py:273] Engine 000: Avg prompt throughput: 6481.6 tokens/s, Avg generation throughput: 371.0 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 24.8%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35342 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35356 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35358 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47932 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47956 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47986 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:47996 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48014 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:19:53 [loggers.py:273] Engine 000: Avg prompt throughput: 16.3 tokens/s, Avg generation throughput: 193.5 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 35.5%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48022 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48048 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48086 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48102 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48154 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48172 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48178 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48196 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48202 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48216 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48226 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48232 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48266 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48278 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48286 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48290 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48294 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48306 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48318 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48320 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48324 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:48332 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35748 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35768 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35782 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35790 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35792 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35798 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35802 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35804 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35834 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35846 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35860 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35874 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:20:03 [loggers.py:273] Engine 000: Avg prompt throughput: 45.0 tokens/s, Avg generation throughput: 646.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 54.1%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35888 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35902 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35910 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35918 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35930 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35946 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35964 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35970 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35980 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:35988 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36002 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36010 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36024 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36038 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36060 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36066 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36072 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36074 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36090 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36100 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36116 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36120 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:36130 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52032 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52064 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52068 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52070 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52076 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52084 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52108 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52118 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:20:13 [loggers.py:273] Engine 000: Avg prompt throughput: 6622.6 tokens/s, Avg generation throughput: 499.2 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 52.5%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52132 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52156 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52166 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52180 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52184 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52198 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52204 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52220 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52236 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52244 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52248 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52258 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52274 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52284 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52300 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52316 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52330 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52334 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52350 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52360 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52374 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52390 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52396 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52400 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52416 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52432 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52446 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52452 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52454 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52466 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52474 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:52482 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57728 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57742 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57752 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57762 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57776 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57788 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57796 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57800 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57808 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57820 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57832 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:20:23 [loggers.py:273] Engine 000: Avg prompt throughput: 10999.7 tokens/s, Avg generation throughput: 475.5 tokens/s, Running: 8 reqs, Waiting: 8 reqs, GPU KV cache usage: 2.0%, Prefix cache hit rate: 45.6%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57836 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57842 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57844 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57852 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57858 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57864 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57880 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57892 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57906 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57914 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57926 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57928 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57940 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57958 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57966 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57974 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57982 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57984 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:57994 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58000 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58004 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58020 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58034 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58046 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:58058 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32944 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32948 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32954 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32962 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32972 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32976 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32978 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:32990 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33006 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33012 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33026 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33028 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33036 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33052 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33056 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO 07-12 15:20:33 [loggers.py:273] Engine 000: Avg prompt throughput: 10684.1 tokens/s, Avg generation throughput: 487.2 tokens/s, Running: 8 reqs, Waiting: 14 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 40.8%
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33078 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33092 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33106 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33114 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33124 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33134 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33150 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
(APIServer pid=2660454) INFO: 127.0.0.1:33158 - "POST /v1/chat/completions HTTP/1.1" 200 OK
|
||||
1
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp2_mns8", "anchor": 0.49609375, "kind": "warmup", "pass_rate": 0.625, "feasible": false}
|
||||
74
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp2_mns8/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.49609375,
|
||||
"cell": "tp2_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 9.026111125,
|
||||
"end_mono_ns": 201115775651265,
|
||||
"end_wall_ns": 1783869542902078194,
|
||||
"start_mono_ns": 201106749540140,
|
||||
"start_wall_ns": 1783869533875967269
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 8,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 0.625,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "818d316f68acc6502ed33a877bf88c2ba9b57a1ff6d6780621db0d30eddf3494",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 3.4938000000000105,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 7,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.13333333333333333,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 7270.0,
|
||||
"min": 71.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "2eb3b78b31efc9354c64c8caa95f08353330553d1ab745a8e0ceab038a3a3d5e",
|
||||
"request_id_order_sha256": "0009512d07ec4c5bb2dd1283c767967a6dfd5bc4c7b7d65fb26169d39a98832e"
|
||||
},
|
||||
"slo_pass_count": 10,
|
||||
"study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece",
|
||||
"tp": 2,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 39.46136329144306,
|
||||
"min": 7.347993740140952,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 4594.941660994664,
|
||||
"min": 762.8166690119542,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.033182214016, "kind": "anchor", "pass_rate": 0.6086956521739131, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.033182214016,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 413,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 49.259978895,
|
||||
"end_mono_ns": 201650764016357,
|
||||
"end_wall_ns": 1783870077890443949,
|
||||
"start_mono_ns": 201601504037462,
|
||||
"start_wall_ns": 1783870028630464875
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 575,
|
||||
"pass_rate": 0.6086956521739131,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "86ee457cd91595c76454d54ebc69890836ef603ceb96deaf9c7cf05e91de9407",
|
||||
"arrival_s": {
|
||||
"distinct_n": 574,
|
||||
"finite_n": 575,
|
||||
"max": 59.90990000000002,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 575
|
||||
},
|
||||
"count": 575,
|
||||
"long_gt4096": 178,
|
||||
"offered_req_s": 9.583333333333334,
|
||||
"offered_req_s_per_gpu": 2.3958333333333335,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 524,
|
||||
"finite_n": 575,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 575
|
||||
},
|
||||
"raw_length_order_sha256": "8354300b1680841ab43ef8164307d7c1f28bc877994b8d1e7584aff030677560",
|
||||
"request_id_order_sha256": "e6b624a0212807df3733c54b4a2160fba24f19ca4776b14f34918d83428bba7f"
|
||||
},
|
||||
"slo_pass_count": 350,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 413,
|
||||
"finite_n": 413,
|
||||
"max": 16.932683684963983,
|
||||
"min": 4.637226771506459,
|
||||
"missing_n": 162,
|
||||
"n": 575
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 413,
|
||||
"finite_n": 413,
|
||||
"max": 4166.508020018227,
|
||||
"min": 38.04149298230186,
|
||||
"missing_n": 162,
|
||||
"n": 575
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.033717411016, "kind": "anchor", "pass_rate": 0.03583617747440273, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.033717411016,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 199,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 41.949754618,
|
||||
"end_mono_ns": 201595581507061,
|
||||
"end_wall_ns": 1783870022707934728,
|
||||
"start_mono_ns": 201553631752443,
|
||||
"start_wall_ns": 1783869980758180054
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 586,
|
||||
"pass_rate": 0.03583617747440273,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "fe44bf6b0efd6e6ab4d19676e34ace608827e419dbb5c19c8595992f0339c9b1",
|
||||
"arrival_s": {
|
||||
"distinct_n": 585,
|
||||
"finite_n": 586,
|
||||
"max": 59.90990000000002,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"count": 586,
|
||||
"long_gt4096": 184,
|
||||
"offered_req_s": 9.766666666666667,
|
||||
"offered_req_s_per_gpu": 2.441666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 534,
|
||||
"finite_n": 586,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"raw_length_order_sha256": "e2d040068b279e99073f6965d1136eb0ca6e8534c883f331b4cb85c5fc7a4650",
|
||||
"request_id_order_sha256": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db"
|
||||
},
|
||||
"slo_pass_count": 21,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 199,
|
||||
"finite_n": 199,
|
||||
"max": 67.64735456709163,
|
||||
"min": 4.931260244077526,
|
||||
"missing_n": 387,
|
||||
"n": 586
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 199,
|
||||
"finite_n": 199,
|
||||
"max": 16151.948870014166,
|
||||
"min": 39.053958986187354,
|
||||
"missing_n": 387,
|
||||
"n": 586
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.034252608017, "kind": "anchor", "pass_rate": 0.8466666666666667, "feasible": false}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.034252608017,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "slo_pass_rate_unrecoverable",
|
||||
"early_stopped": true,
|
||||
"exact_output_count": 581,
|
||||
"feasible": false,
|
||||
"interval": {
|
||||
"elapsed_s": 63.947365009,
|
||||
"end_mono_ns": 201732794742162,
|
||||
"end_wall_ns": 1783870159921169450,
|
||||
"start_mono_ns": 201668847377153,
|
||||
"start_wall_ns": 1783870095973805093
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 600,
|
||||
"pass_rate": 0.8466666666666667,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "50f1b34c344c451c6d7501290d0b53f147d0c0b6a4bc078b303d5c33576ebe2d",
|
||||
"arrival_s": {
|
||||
"distinct_n": 599,
|
||||
"finite_n": 600,
|
||||
"max": 59.90990000000002,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 600
|
||||
},
|
||||
"count": 600,
|
||||
"long_gt4096": 187,
|
||||
"offered_req_s": 10.0,
|
||||
"offered_req_s_per_gpu": 2.5,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 546,
|
||||
"finite_n": 600,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 600
|
||||
},
|
||||
"raw_length_order_sha256": "08eff8eecd74dc4646aa45945bd752432ee5eb795006755527dd32d7178c1140",
|
||||
"request_id_order_sha256": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd"
|
||||
},
|
||||
"slo_pass_count": 508,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 581,
|
||||
"finite_n": 581,
|
||||
"max": 17.87731141735631,
|
||||
"min": 4.250030724219904,
|
||||
"missing_n": 19,
|
||||
"n": 600
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 581,
|
||||
"finite_n": 581,
|
||||
"max": 4941.07082701521,
|
||||
"min": 36.84570998302661,
|
||||
"missing_n": 19,
|
||||
"n": 600
|
||||
}
|
||||
}
|
||||
22
runs/opprof-phase6/phase6/cells/tp4_mns16/cell-valid.json
Normal file
22
runs/opprof-phase6/phase6/cells/tp4_mns16/cell-valid.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"accounting_mode": "graceful-footer",
|
||||
"cell": "tp4_mns16",
|
||||
"invariants": {
|
||||
"anchor_intervals_present": true,
|
||||
"compile_capture_pre_ready": true,
|
||||
"encoded_balanced": true,
|
||||
"footer_sidecar_agrees": true,
|
||||
"last_step_matches": true,
|
||||
"layer1_contiguous": true,
|
||||
"layer1_zero_drops": true,
|
||||
"one_footer_last": true,
|
||||
"one_ready_marker": true,
|
||||
"sidecar_final": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true,
|
||||
"written_matches_records": true
|
||||
},
|
||||
"layer1_records": 26137,
|
||||
"post_ready_capture_events": [],
|
||||
"stream": "/home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/opprof/opprof-v1-dp0-pid2670831-1783869959685052499.jsonl"
|
||||
}
|
||||
6
runs/opprof-phase6/phase6/cells/tp4_mns16/commands.log
Normal file
6
runs/opprof-phase6/phase6/cells/tp4_mns16/commands.log
Normal file
@@ -0,0 +1,6 @@
|
||||
SERVER taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/vllm serve /home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B --host 127.0.0.1 --port 8521 --served-model-name qwen3-30b-a3b-community --max-num-batched-tokens 8192 --max-num-seqs 16 --tensor-parallel-size 4 --shutdown-timeout 120
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py warmup --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.033717411016 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/warmup
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.033717411016 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/anchor-0.033717411016
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.033182214016 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/anchor-0.033182214016
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.034252608017 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/anchor-0.034252608017
|
||||
CLIENT taskset -c 80-99,100-119,120-139,140-159 /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/scripts/opprof_phase6_client.py run-anchor --study /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/provenance/study-tp4.json --cell tp4_mns16 --anchor 0.033717411016 --tp 4 --mns 16 --base-url http://127.0.0.1:8521 --result-dir /home/admin/cpfs/wjh/opprof-phase6-dash0-20260712/runs/phase6/cells/tp4_mns16/confirm-1-anchor-0.033717411016
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.033717411016, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.033717411016,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 586,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.364052595,
|
||||
"end_mono_ns": 201800328727441,
|
||||
"end_wall_ns": 1783870227455155582,
|
||||
"start_mono_ns": 201738964674846,
|
||||
"start_wall_ns": 1783870166091102450
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 16,
|
||||
"observed_count": 586,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "fe44bf6b0efd6e6ab4d19676e34ace608827e419dbb5c19c8595992f0339c9b1",
|
||||
"arrival_s": {
|
||||
"distinct_n": 585,
|
||||
"finite_n": 586,
|
||||
"max": 59.90990000000002,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"count": 586,
|
||||
"long_gt4096": 184,
|
||||
"offered_req_s": 9.766666666666667,
|
||||
"offered_req_s_per_gpu": 2.441666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 534,
|
||||
"finite_n": 586,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"raw_length_order_sha256": "e2d040068b279e99073f6965d1136eb0ca6e8534c883f331b4cb85c5fc7a4650",
|
||||
"request_id_order_sha256": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db"
|
||||
},
|
||||
"slo_pass_count": 586,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 586,
|
||||
"finite_n": 586,
|
||||
"max": 14.420212055125647,
|
||||
"min": 4.217363204660378,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 586,
|
||||
"finite_n": 586,
|
||||
"max": 819.4046519929543,
|
||||
"min": 36.395939998328686,
|
||||
"missing_n": 0,
|
||||
"n": 586
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"schema":1,"record_type":"footer_checkpoint","stream":"opprof-v1-dp0-pid2670831-1783869959685052499.jsonl","encoded_records":26137,"written_records":26137,"dropped_records":0,"last_step_index":26136,"checkpoint_wall_ns":1783870239366621112,"flush_interval_seconds":1.0,"final":true}
|
||||
2003
runs/opprof-phase6/phase6/cells/tp4_mns16/server.log
Normal file
2003
runs/opprof-phase6/phase6/cells/tp4_mns16/server.log
Normal file
File diff suppressed because it is too large
Load Diff
1
runs/opprof-phase6/phase6/cells/tp4_mns16/warmup.log
Normal file
1
runs/opprof-phase6/phase6/cells/tp4_mns16/warmup.log
Normal file
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns16", "anchor": 0.033717411016, "kind": "warmup", "pass_rate": 1.0, "feasible": true}
|
||||
74
runs/opprof-phase6/phase6/cells/tp4_mns16/warmup/result.json
Normal file
74
runs/opprof-phase6/phase6/cells/tp4_mns16/warmup/result.json
Normal file
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.033717411016,
|
||||
"cell": "tp4_mns16",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 16,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 3.923353377,
|
||||
"end_mono_ns": 201544201950878,
|
||||
"end_wall_ns": 1783869971328377792,
|
||||
"start_mono_ns": 201540278597501,
|
||||
"start_wall_ns": 1783869967405025053
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "warmup",
|
||||
"mns": 16,
|
||||
"observed_count": 16,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "90b5197b9bfe4a9d6dfa881ec683b528ba46dd7bdc3a42fe2c91f6aefc6c8308",
|
||||
"arrival_s": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 1.7244999999999893,
|
||||
"min": 0.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"count": 16,
|
||||
"long_gt4096": 2,
|
||||
"offered_req_s": 0.26666666666666666,
|
||||
"offered_req_s_per_gpu": 0.06666666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 8150.0,
|
||||
"min": 371.0,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"raw_length_order_sha256": "d416a666bb3b2d9a0f1ffe0c51cc4e41249123c02ecb4eadb6fbb049422d7e3d",
|
||||
"request_id_order_sha256": "01b7744965c81da433fd52073d45f36d4b5bc750ca0bbd5ab0db8d8eac316fa3"
|
||||
},
|
||||
"slo_pass_count": 16,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 21.061303456796068,
|
||||
"min": 6.4699021653880875,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 16,
|
||||
"finite_n": 16,
|
||||
"max": 1454.2098789825104,
|
||||
"min": 132.21911297296174,
|
||||
"missing_n": 0,
|
||||
"n": 16
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"cell": "tp4_mns8", "anchor": 0.016055910008, "kind": "anchor", "pass_rate": 1.0, "feasible": true}
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"anchor": 0.016055910008,
|
||||
"cell": "tp4_mns8",
|
||||
"early_stop_reason": "",
|
||||
"early_stopped": false,
|
||||
"exact_output_count": 301,
|
||||
"feasible": true,
|
||||
"interval": {
|
||||
"elapsed_s": 61.26476028,
|
||||
"end_mono_ns": 201662804315076,
|
||||
"end_wall_ns": 1783870089930742071,
|
||||
"start_mono_ns": 201601539554796,
|
||||
"start_wall_ns": 1783870028665982142
|
||||
},
|
||||
"invariants": {
|
||||
"arrival_nondecreasing": true,
|
||||
"exact_output_or_failed": true,
|
||||
"outcomes_cover_selected": true,
|
||||
"raw_lengths_present": true,
|
||||
"selected_nonempty": true,
|
||||
"warmup_16": true,
|
||||
"warmup_exact_16": true,
|
||||
"warmup_long": true
|
||||
},
|
||||
"kind": "anchor",
|
||||
"mns": 8,
|
||||
"observed_count": 301,
|
||||
"pass_rate": 1.0,
|
||||
"schema": 1,
|
||||
"selection": {
|
||||
"arrival_order_sha256": "42c88c970510ae1b49ebcf10d2a9fb9bbc07c2ec2761dfd19193a6962cab1273",
|
||||
"arrival_s": {
|
||||
"distinct_n": 300,
|
||||
"finite_n": 301,
|
||||
"max": 59.75439999999999,
|
||||
"min": 0.13000000000001818,
|
||||
"missing_n": 0,
|
||||
"n": 301
|
||||
},
|
||||
"count": 301,
|
||||
"long_gt4096": 95,
|
||||
"offered_req_s": 5.016666666666667,
|
||||
"offered_req_s_per_gpu": 1.2541666666666667,
|
||||
"raw_input_tokens": {
|
||||
"distinct_n": 283,
|
||||
"finite_n": 301,
|
||||
"max": 8157.0,
|
||||
"min": 69.0,
|
||||
"missing_n": 0,
|
||||
"n": 301
|
||||
},
|
||||
"raw_length_order_sha256": "d1765a9ae2785d90f135e114ce1d802dec58e86fd36152182dee987a73d1604c",
|
||||
"request_id_order_sha256": "2d680e08c068237fadfff567b3fb7f55a8e0a0121c009cefa07ee2653093e700"
|
||||
},
|
||||
"slo_pass_count": 301,
|
||||
"study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26",
|
||||
"tp": 4,
|
||||
"tpot_ms": {
|
||||
"distinct_n": 301,
|
||||
"finite_n": 301,
|
||||
"max": 12.61807709458416,
|
||||
"min": 3.798431708609436,
|
||||
"missing_n": 0,
|
||||
"n": 301
|
||||
},
|
||||
"ttft_ms": {
|
||||
"distinct_n": 301,
|
||||
"finite_n": 301,
|
||||
"max": 1491.487142018741,
|
||||
"min": 37.34235602314584,
|
||||
"missing_n": 0,
|
||||
"n": 301
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user